Branch data Line data Source code
1 : : #include "precizer.h"
2 : : #include <errno.h>
3 : : #include <fcntl.h>
4 : :
5 : : /*
6 : : * Minimum elapsed monotonic time between periodic hash checkpoints.
7 : : * The value is stored in nanoseconds; 15 seconds bounds worst-case lost hashing progress after an unexpected crash without writing to SQLite too often
8 : : */
9 : : static const long long int sha512_checkpoint_interval_ns = 14930016475LL;
10 : :
11 : : /**
12 : : * @brief Check whether periodic database checkpoints are safe for this hash pass
13 : : *
14 : : * Periodic checkpoints are skipped in dry-run mode because no persistent DB
15 : : * state should be changed. They are also skipped for a fully sealed
16 : : * checksum-locked file, so no control path can replace the stored final
17 : : * checksum with a partial hash state
18 : : *
19 : : * @param[in] file Per-file state currently being hashed
20 : : * @return true when a periodic checkpoint may be written, otherwise false
21 : : */
22 : 5 : static bool periodic_hash_checkpoint_is_allowed(const File *file)
23 : : {
24 [ - + ]: 5 : if(config->dry_run == true)
25 : : {
26 : 0 : return false;
27 : : }
28 : :
29 [ - + ]: 5 : if(file->checksum_offset <= 0)
30 : : {
31 : 0 : return false;
32 : : }
33 : :
34 : : /*
35 : : * A sealed checksum-locked row must keep its trusted final checksum and metadata.
36 : : * Even if a future control path reaches sha512sum(), periodic checkpoints must
37 : : * not replace it with temporary offset/mdContext state
38 : : */
39 [ + + ]: 5 : if(file->locked_checksum_file == true
40 [ + - ]: 1 : && file->db->relative_path_was_in_db_before_processing == true
41 [ + - ]: 1 : && file->db->saved_offset == 0)
42 : : {
43 : 1 : return false;
44 : : }
45 : :
46 : 4 : return true;
47 : : }
48 : :
49 : : /**
50 : : * @brief Read a file and update its SHA512 state when hashing is enabled
51 : : *
52 : : * Opens @p relative_path from @p root_directory_fd. When normal hashing is
53 : : * enabled, or when dry-run uses `--dry-run=with-checksums`, the function reads
54 : : * file data starting from @p file->checksum_offset, updates @p file->mdContext,
55 : : * counts hashed bytes in @p summary, and finalizes @p file->sha512 after an
56 : : * uninterrupted pass. In dry-run mode without checksum calculation, the file is
57 : : * opened and seek-checked but the checksum state is not advanced
58 : : *
59 : : * File opening problems are stored in @p file as read errors and do not turn
60 : : * into a function failure. Technical problems that prevent safe hashing still
61 : : * return FAILURE
62 : : *
63 : : * @param root_directory_fd Open traversal root descriptor used as the path base
64 : : * @param relative_path File path relative to @p root_directory_fd
65 : : * @param file_buffer Read buffer descriptor
66 : : * @param summary Traversal counters updated with hashed byte count and hashing time
67 : : * @param file Per-file state object used as input and output. checksum_offset is the
68 : : * starting byte offset for resumption and is updated as bytes are
69 : : * hashed. sha512 receives the final digest. mdContext holds the
70 : : * incremental hashing state. read_error and read_errno describe
71 : : * read failures. wrong_file_type is set for non-seekable or
72 : : * otherwise unsupported file types
73 : : * @param[in,out] path_known True when the row currently exists in SQLite.
74 : : * Periodic checkpoints may flip this to true after
75 : : * inserting a partial row for a previously unknown path
76 : : * @return SUCCESS when the file was handled cleanly, otherwise FAILURE
77 : : */
78 : 2603 : Return sha512sum(
79 : : const int root_directory_fd,
80 : : const memory *relative_path,
81 : : memory *file_buffer,
82 : : TraversalSummary *summary,
83 : : File *file,
84 : : bool *path_known)
85 : : {
86 : : /* Status returned by this function through provide()
87 : : Default value assumes successful completion */
88 : 2603 : Return status = SUCCESS;
89 : :
90 [ + - - + ]: 2603 : if(file_buffer->length == 0 || path_known == NULL)
91 : : {
92 : 0 : slog(ERROR,"Invalid buffer size: %ld bytes\n",file_buffer->length);
93 : 0 : provide(FAILURE);
94 : : }
95 : :
96 : 2603 : const char *runtime_relative_path = m_text(relative_path);
97 : :
98 : 2603 : const int file_descriptor = openat(root_directory_fd,runtime_relative_path,O_RDONLY | O_CLOEXEC);
99 : :
100 [ + + ]: 2603 : if(file_descriptor < 0)
101 : : {
102 : : // Flag the read failure
103 : 1 : file->read_error = true;
104 : :
105 : : // Preserve errno before returning
106 : 1 : file->read_errno = errno;
107 : :
108 : 1 : provide(status);
109 : : }
110 : :
111 : 2602 : FILE *fileptr = fdopen(file_descriptor,"rb");
112 : :
113 [ + + ]: 2602 : if(fileptr == NULL)
114 : : {
115 : 1 : const int fdopen_errno = errno;
116 : :
117 [ - + ]: 1 : if(close(file_descriptor) != 0)
118 : : {
119 : 0 : slog(ERROR,"Error closing file descriptor for %s\n",runtime_relative_path);
120 : : }
121 : :
122 : 1 : file->read_error = true;
123 : 1 : file->read_errno = fdopen_errno;
124 : :
125 : 1 : provide(status);
126 : : }
127 : :
128 : : // Move the file pointer checksum_offset bytes from the beginning of the file
129 [ - + ]: 2601 : if(fseek(fileptr,file->checksum_offset,SEEK_SET) != 0)
130 : : {
131 : : /*
132 : : * This looks like an unsupported file type.
133 : : * Doesn't need to return FAILURE status.
134 : : */
135 : 0 : file->wrong_file_type = true;
136 : 0 : fclose(fileptr);
137 : 0 : provide(status);
138 : : }
139 : :
140 : 2601 : bool loop_was_interrupted = false;
141 : 5202 : bool perform_file_hashing = config->dry_run == false
142 [ + + + + ]: 2601 : || config->dry_run_with_checksums == true;
143 : :
144 : : /*
145 : : * Shared random-stop state for SHA512 test hooks.
146 : : * Test builds use these values to coordinate byte-exact interruption,
147 : : * byte-exact checkpointing, and the optional crash-after-checkpoint scenario
148 : : */
149 : : #ifdef TESTITALL_TEST_HOOKS
150 : : /*
151 : : * 0 means random-stop flow is disabled for this file.
152 : : * Non-zero means upper bound for random stop byte selection.
153 : : */
154 : 2601 : uint64_t random_stop_limit = 0U;
155 : : /* 0 means stop byte has not been selected yet. */
156 : 2601 : uint64_t random_stop_byte_value = 0U;
157 : : /* Separate state flag: do not overload stop-byte numeric value. */
158 : 2601 : bool random_stop_triggered = false;
159 : 2601 : bool testitall_checkpoint_at_random_byte = testitall_hash_checkpoint_at_random_byte_enabled();
160 : : #endif
161 : :
162 [ + + ]: 2601 : if(file->checksum_offset == 0)
163 : : {
164 : : // Fresh hashing pass: initialize the SHA512 state from scratch
165 [ - + ]: 2596 : if(sha512_init(&file->mdContext) != CRYPT_OK)
166 : : {
167 : 0 : slog(ERROR,"SHA512 initialization failed\n");
168 : 0 : fclose(fileptr);
169 : 0 : provide(FAILURE);
170 : : }
171 : : }
172 : :
173 : : /*
174 : : * Test hook A: choose the byte where the test should interfere.
175 : : * Later hook comments refer to hook A because signal waiting, read limiting,
176 : : * and byte-forced checkpoints all need the same exact target byte
177 : : */
178 : : #ifdef TESTITALL_TEST_HOOKS
179 : : /*
180 : : * Activate random interruption for a fresh pass of hugetestfile.
181 : : * Byte-checkpoint tests may also select a stop point after resume, so
182 : : * checkpoint-update paths can be exercised without a signal
183 : : */
184 [ + + + + ]: 2601 : if((file->checksum_offset == 0 || testitall_checkpoint_at_random_byte == true)
185 [ + + ]: 2597 : && testitall_is_huge_interruption_target(runtime_relative_path) == true
186 [ + - ]: 11 : && file->stat.st_size > 0)
187 : : {
188 : 11 : const uint64_t file_size = (uint64_t)file->stat.st_size;
189 : 11 : uint64_t current_offset = 0U;
190 : :
191 [ + + ]: 11 : if(file->checksum_offset > 0)
192 : : {
193 : 1 : current_offset = (uint64_t)file->checksum_offset;
194 : : }
195 : :
196 [ + - ]: 11 : if(current_offset < file_size)
197 : : {
198 : 11 : const uint64_t remaining_file_size = file_size - current_offset;
199 : :
200 [ + + - + ]: 11 : if(testitall_checkpoint_at_random_byte == true && remaining_file_size <= 1U)
201 : : {
202 : 0 : random_stop_limit = 0U;
203 : :
204 : : } else {
205 : 11 : random_stop_limit = file_size;
206 : :
207 : : /*
208 : : * Select interruption target before the first fread() call so
209 : : * the first chunk can be bounded and the stop point can land
210 : : * anywhere in the remaining file range.
211 : : */
212 : 11 : random_stop_byte_value = current_offset + testitall_random_stop_byte(remaining_file_size);
213 : :
214 : : /* Defensive fallback: never allow a zero stop byte. */
215 [ - + ]: 11 : if(random_stop_byte_value == 0U)
216 : : {
217 : 0 : random_stop_byte_value = 1U;
218 : : }
219 : :
220 : : /*
221 : : * Keep the first byte-checkpoint stop at least two bytes before EOF.
222 : : * This leaves room for the resume pass to write another checkpoint
223 : : * before the final hash state is reached
224 : : */
225 [ + + ]: 11 : if(testitall_checkpoint_at_random_byte == true
226 [ + + ]: 5 : && current_offset == 0U
227 [ + - ]: 4 : && random_stop_limit > 2U
228 [ - + ]: 4 : && random_stop_byte_value >= random_stop_limit - 1U)
229 : : {
230 : 0 : random_stop_byte_value = random_stop_limit - 2U;
231 : :
232 [ + - - + ]: 11 : } else if(random_stop_limit > 1U && random_stop_byte_value >= random_stop_limit){
233 : : /*
234 : : * Keep the stop point strictly inside file data for multi-byte files.
235 : : * If random selection lands exactly at EOF, shift it one byte left.
236 : : * The guard keeps subtraction safe and avoids unsigned underflow
237 : : */
238 : 0 : random_stop_byte_value = random_stop_limit - 1U;
239 : : }
240 : :
241 [ - + - - ]: 11 : if(random_stop_byte_value <= current_offset && current_offset + 1U < random_stop_limit)
242 : : {
243 : 0 : random_stop_byte_value = current_offset + 1U;
244 : : }
245 : : }
246 : : }
247 : : }
248 : : #endif
249 : :
250 [ + + ]: 2601 : if(perform_file_hashing == true)
251 : : {
252 : 2541 : long long int hashing_start_ns = cur_time_monotonic_ns();
253 : :
254 : : /*
255 : : * Monotonic timestamp when the next periodic checkpoint should be considered.
256 : : * It is advanced after each checkpoint window so the database is not updated on every read block
257 : : */
258 : 2541 : long long int next_checkpoint_ns = hashing_start_ns + sha512_checkpoint_interval_ns;
259 : :
260 : 2541 : unsigned char *file_buffer_data_rewritable = m_raw_data(file_buffer);
261 : :
262 : : while(true)
263 : 2542 : {
264 : : /*
265 : : * Test hook B: notify the signal-driven interruption test after the byte
266 : : * selected by hook A has been hashed. The delay hook uses this as the
267 : : * proof that the controlled stop point was really reached
268 : : */
269 : : #ifdef TESTITALL_TEST_HOOKS
270 : : /*
271 : : * Trigger point 2 exactly once when selected stop byte is reached.
272 : : */
273 [ + + ]: 5083 : if(random_stop_limit > 0U
274 [ + + ]: 25 : && testitall_checkpoint_at_random_byte == false
275 [ + + ]: 16 : && random_stop_triggered == false
276 [ + - ]: 12 : && random_stop_byte_value > 0U
277 [ + + ]: 12 : && (uint64_t)(file->checksum_offset) >= random_stop_byte_value)
278 : : {
279 : 6 : signal_wait_at_point(2U);
280 : 6 : random_stop_triggered = true;
281 : : }
282 : : #endif
283 : :
284 : : /* Interrupt the loop smoothly */
285 : : /* Interrupt when Ctrl+C */
286 : : /*
287 : : * Test hook C: decide whether normal Ctrl+C/SIGINT handling must wait.
288 : : * This protects the signal-driven random-stop test until hook B has
289 : : * reached the byte selected by hook A
290 : : */
291 : : #ifdef TESTITALL_TEST_HOOKS
292 : : /*
293 : : * Test-only guard: when random-stop mode is active for hugetestfile,
294 : : * do not break on global_interrupt_flag until point 2 has really
295 : : * happened. Otherwise interruption may fire too early and miss the
296 : : * controlled "interrupt at random byte" scenario.
297 : : */
298 : 5083 : bool delay_interrupt_for_random_stop = false;
299 : :
300 [ + + ]: 5083 : if(random_stop_limit > 0U
301 [ + + ]: 25 : && testitall_checkpoint_at_random_byte == false
302 [ + + ]: 16 : && random_stop_triggered == false)
303 : : {
304 [ - + ]: 6 : if(random_stop_byte_value == 0U)
305 : : {
306 : : /* No stop byte yet: wait until at least one block is hashed. */
307 : 0 : delay_interrupt_for_random_stop = true;
308 : :
309 [ + - ]: 6 : } else if((uint64_t)(file->checksum_offset) < random_stop_byte_value){
310 : : /* Stop byte is known but not reached yet: keep hashing. */
311 : 6 : delay_interrupt_for_random_stop = true;
312 : : }
313 : : }
314 : : #endif
315 : :
316 : : /*
317 : : * The normal interrupt condition also honors hook C in test builds.
318 : : * This keeps production behavior intact while letting the test reach
319 : : * the controlled stop byte before breaking the read loop
320 : : */
321 [ + + ]: 5083 : if(global_interrupt_flag == true
322 : : #ifdef TESTITALL_TEST_HOOKS
323 [ + - ]: 2 : && delay_interrupt_for_random_stop == false
324 : : #endif
325 : : )
326 : : {
327 : 2 : loop_was_interrupted = true;
328 : 2 : break;
329 : : }
330 : :
331 : 5081 : size_t read_limit = file_buffer->length;
332 : :
333 : : /*
334 : : * Cap one fread() call so the loop lands exactly on the byte selected
335 : : * by hook A. Without this cap, a large read could jump past the target
336 : : * and make the checkpoint/resume test flaky
337 : : */
338 : : #ifdef TESTITALL_TEST_HOOKS
339 : : /*
340 : : * Keep the read bounded so offset can stop exactly at the selected
341 : : * byte instead of jumping to EOF in a single large fread().
342 : : */
343 [ + + ]: 5081 : if(random_stop_limit > 0U
344 [ + + ]: 23 : && random_stop_triggered == false
345 [ + - ]: 11 : && random_stop_byte_value > 0U
346 [ + - ]: 11 : && (uint64_t)(file->checksum_offset) < random_stop_byte_value)
347 : : {
348 : 11 : const uint64_t bytes_left_to_stop = random_stop_byte_value - (uint64_t)(file->checksum_offset);
349 : :
350 [ + - ]: 11 : if(bytes_left_to_stop < (uint64_t)read_limit)
351 : : {
352 : 11 : read_limit = (size_t)bytes_left_to_stop;
353 : : }
354 : : }
355 : : #endif
356 : :
357 : 5081 : size_t len = fread(file_buffer_data_rewritable,sizeof(unsigned char),read_limit,fileptr);
358 : :
359 [ + + ]: 5081 : if(len == 0)
360 : : {
361 [ + + ]: 2536 : if(ferror(fileptr))
362 : : {
363 : 1 : file->read_error = true;
364 : 1 : file->read_errno = errno;
365 : : }
366 : :
367 : 2536 : break;
368 : : }
369 : :
370 [ + - ]: 2545 : if(SUCCESS == status)
371 : : {
372 [ - + ]: 2545 : if(sha512_update(&file->mdContext,file_buffer_data_rewritable,len) != CRYPT_OK)
373 : : {
374 : 0 : slog(ERROR,"SHA512 update failed\n");
375 : 0 : status = FAILURE;
376 : 0 : break;
377 : : }
378 : :
379 : 2545 : file->checksum_offset += (sqlite3_int64)len;
380 : 2545 : summary->total_hashed_bytes += len;
381 : :
382 : : /*
383 : : * Test hook D: request an immediate checkpoint when the byte selected
384 : : * by hook A has just been hashed. The checkpoint condition and crash
385 : : * simulation use this flag to avoid waiting for the time interval
386 : : */
387 : : #ifdef TESTITALL_TEST_HOOKS
388 : 2545 : bool testitall_checkpoint_now = false;
389 : :
390 [ + + ]: 2545 : if(testitall_checkpoint_at_random_byte == true
391 [ + - ]: 7 : && random_stop_limit > 0U
392 [ + + ]: 7 : && random_stop_triggered == false
393 [ + - ]: 5 : && random_stop_byte_value > 0U
394 [ + - ]: 5 : && (uint64_t)(file->checksum_offset) >= random_stop_byte_value)
395 : : {
396 : 5 : testitall_checkpoint_now = true;
397 : 5 : random_stop_triggered = true;
398 : : }
399 : : #endif
400 : :
401 : : /*
402 : : * Current monotonic timestamp sampled after the SHA512 state and byte offset were advanced.
403 : : * Saving only after this point keeps offset and mdContext consistent in the database
404 : : */
405 : 2545 : const long long int checkpoint_now_ns = cur_time_monotonic_ns();
406 : :
407 : : /*
408 : : * Honor hook D by entering the checkpoint path immediately.
409 : : * This is what makes byte-precise checkpoint tests practical even
410 : : * when the normal time-based checkpoint interval has not elapsed
411 : : */
412 [ + - ]: 2545 : if(checkpoint_now_ns >= next_checkpoint_ns
413 : : #ifdef TESTITALL_TEST_HOOKS
414 [ + + ]: 2545 : || testitall_checkpoint_now == true
415 : : #endif
416 : : )
417 : : {
418 : : /*
419 : : * Test hook E: remember whether this checkpoint really reached
420 : : * the database. The crash simulation relies on hook E so it only
421 : : * exits after there is durable state to resume from
422 : : */
423 : : #ifdef TESTITALL_TEST_HOOKS
424 : 5 : bool checkpoint_saved = false;
425 : : #endif
426 : :
427 [ + + ]: 5 : if(periodic_hash_checkpoint_is_allowed(file) == true)
428 : : {
429 : 4 : status = db_save_file_record(relative_path,file,path_known,false);
430 : :
431 [ - + ]: 4 : if((TRIUMPH & status) == 0)
432 : : {
433 : 0 : break;
434 : : }
435 : :
436 : : /*
437 : : * Set hook E only after db_save_file_record() succeeds.
438 : : * This connects the real DB write above with the crash
439 : : * simulation below
440 : : */
441 : : #ifdef TESTITALL_TEST_HOOKS
442 : 4 : checkpoint_saved = true;
443 : : #endif
444 : : }
445 : :
446 : : /*
447 : : * Simulate a sudden process death only when hook D requested this
448 : : * byte checkpoint and hook E proves it was saved. The next test run
449 : : * can then prove that the stored partial SHA512 state survived
450 : : */
451 : : #ifdef TESTITALL_TEST_HOOKS
452 [ + + ]: 5 : if(checkpoint_saved == true
453 [ + - ]: 4 : && testitall_checkpoint_now == true
454 [ + + ]: 4 : && testitall_exit_after_hash_checkpoint_enabled() == true)
455 : : {
456 : 3 : testitall_exit_after_hash_checkpoint();
457 : : }
458 : : #endif
459 : :
460 : 2 : next_checkpoint_ns = checkpoint_now_ns + sha512_checkpoint_interval_ns;
461 : : }
462 : : }
463 : : }
464 : :
465 : 2538 : long long int hashing_stop_ns = cur_time_monotonic_ns();
466 : :
467 : 2538 : long long int hashing_elapsed_ns = hashing_stop_ns - hashing_start_ns;
468 : :
469 [ - + ]: 2538 : if(hashing_elapsed_ns < 0LL)
470 : : {
471 : 0 : hashing_elapsed_ns = 0LL;
472 : : }
473 : :
474 : 2538 : summary->total_hashing_elapsed_ns += hashing_elapsed_ns;
475 : : }
476 : :
477 [ - + ]: 2598 : if(fclose(fileptr) != 0)
478 : : {
479 : 0 : slog(ERROR,"Error closing file %s\n",runtime_relative_path);
480 : : }
481 : :
482 [ + - ]: 2598 : if(SUCCESS == status
483 [ + + ]: 2598 : && perform_file_hashing == true
484 [ + + ]: 2538 : && loop_was_interrupted == false)
485 : : {
486 : 2536 : file->checksum_offset = 0;
487 : :
488 [ - + ]: 2536 : if(sha512_final(&file->mdContext,file->sha512) != CRYPT_OK)
489 : : {
490 : 0 : slog(ERROR,"SHA512 finalization failed\n");
491 : 0 : status = FAILURE;
492 : : }
493 : : }
494 : :
495 : 2598 : provide(status);
496 : : }
|
