Branch data Line data Source code
1 : : #include "precizer.h"
2 : : #include <errno.h>
3 : : #include <fcntl.h>
4 : :
5 : : /*
6 : : * Periodic hash checkpoints use the shared DB_CHECKPOINT_INTERVAL_NS interval.
7 : : * Pending file batches can bring a checkpoint forward so earlier writes do not wait for another full interval
8 : : */
9 : :
10 : : /**
11 : : * @brief Check whether periodic database checkpoints are safe for this hash pass
12 : : *
13 : : * Periodic checkpoints are skipped in dry-run mode because no persistent DB
14 : : * state should be changed. They are also skipped for a fully sealed
15 : : * checksum-locked file, so no control path can replace the stored final
16 : : * checksum with a partial hash state
17 : : *
18 : : * @param[in] file Per-file state currently being hashed
19 : : * @return true when a periodic checkpoint may be written, otherwise false
20 : : */
21 : 5 : static bool periodic_hash_checkpoint_is_allowed(const File *file)
22 : : {
23 [ - + ]: 5 : if(config->dry_run == true)
24 : : {
25 : 0 : return false;
26 : : }
27 : :
28 [ - + ]: 5 : if(file->checksum_offset <= 0)
29 : : {
30 : 0 : return false;
31 : : }
32 : :
33 : : /*
34 : : * A sealed checksum-locked row must keep its trusted final checksum and metadata.
35 : : * Even if a future control path reaches sha512sum(), periodic checkpoints must
36 : : * not replace it with temporary offset/mdContext state
37 : : */
38 [ + + ]: 5 : if(file->locked_checksum_file == true
39 [ + - ]: 1 : && file->db->relative_path_was_in_db_before_processing == true
40 [ + - ]: 1 : && file->db->saved_offset == 0)
41 : : {
42 : 1 : return false;
43 : : }
44 : :
45 : 4 : return true;
46 : : }
47 : :
48 : : /**
49 : : * @brief Read a file and update its SHA512 state when hashing is enabled
50 : : *
51 : : * Opens @p relative_path from @p root_directory_fd. When normal hashing is
52 : : * enabled, or when dry-run uses `--dry-run=with-checksums`, the function reads
53 : : * file data starting from @p file->checksum_offset, updates @p file->mdContext,
54 : : * counts hashed bytes in @p summary, and finalizes @p file->sha512 after an
55 : : * uninterrupted pass. In dry-run mode without checksum calculation, the file is
56 : : * opened and seek-checked but the checksum state is not advanced
57 : : *
58 : : * File opening problems are stored in @p file as read errors and do not turn
59 : : * into a function failure. Technical problems that prevent safe hashing still
60 : : * return FAILURE
61 : : *
62 : : * @param root_directory_fd Open traversal root descriptor used as the path base
63 : : * @param relative_path File path relative to @p root_directory_fd
64 : : * @param file_buffer Read buffer descriptor
65 : : * @param summary Traversal counters updated with hashed byte count and hashing time
66 : : * @param file Per-file state object used as input and output. checksum_offset is the
67 : : * starting byte offset for resumption and is updated as bytes are
68 : : * hashed. sha512 receives the final digest. mdContext holds the
69 : : * incremental hashing state. read_error and read_errno describe
70 : : * read failures. wrong_file_type is set for non-seekable or
71 : : * otherwise unsupported file types
72 : : * @param[in,out] path_known True when the row currently exists in SQLite.
73 : : * Periodic checkpoints may flip this to true after
74 : : * inserting a partial row for a previously unknown path
75 : : * @return SUCCESS when the file was handled cleanly, otherwise FAILURE
76 : : */
77 : 3613 : Return sha512sum(
78 : : const int root_directory_fd,
79 : : const memory *relative_path,
80 : : memory *file_buffer,
81 : : TraversalSummary *summary,
82 : : File *file,
83 : : bool *path_known)
84 : : {
85 : : /* Status returned by this function through provide()
86 : : Default value assumes successful completion */
87 : 3613 : Return status = SUCCESS;
88 : :
89 [ + - - + ]: 3613 : if(file_buffer->length == 0 || path_known == NULL)
90 : : {
91 : 0 : slog(ERROR,"Invalid buffer size: %ld bytes\n",file_buffer->length);
92 : 0 : provide(FAILURE);
93 : : }
94 : :
95 : 3613 : const char *runtime_relative_path = m_text(relative_path);
96 : :
97 : 3613 : const int file_descriptor = openat(root_directory_fd,runtime_relative_path,O_RDONLY | O_CLOEXEC);
98 : :
99 [ + + ]: 3613 : if(file_descriptor < 0)
100 : : {
101 : : // Flag the read failure
102 : 1 : file->read_error = true;
103 : :
104 : : // Preserve errno before returning
105 : 1 : file->read_errno = errno;
106 : :
107 : 1 : provide(status);
108 : : }
109 : :
110 : 3612 : FILE *fileptr = fdopen(file_descriptor,"rb");
111 : :
112 [ + + ]: 3612 : if(fileptr == NULL)
113 : : {
114 : 1 : const int fdopen_errno = errno;
115 : :
116 [ - + ]: 1 : if(close(file_descriptor) != 0)
117 : : {
118 : 0 : slog(ERROR,"Error closing file descriptor for %s\n",runtime_relative_path);
119 : : }
120 : :
121 : 1 : file->read_error = true;
122 : 1 : file->read_errno = fdopen_errno;
123 : :
124 : 1 : provide(status);
125 : : }
126 : :
127 : : // Move the file pointer checksum_offset bytes from the beginning of the file
128 [ - + ]: 3611 : if(fseek(fileptr,file->checksum_offset,SEEK_SET) != 0)
129 : : {
130 : : /*
131 : : * This looks like an unsupported file type.
132 : : * Doesn't need to return FAILURE status.
133 : : */
134 : 0 : file->wrong_file_type = true;
135 : 0 : fclose(fileptr);
136 : 0 : provide(status);
137 : : }
138 : :
139 : 3611 : bool loop_was_interrupted = false;
140 : 7222 : bool perform_file_hashing = config->dry_run == false
141 [ + + + + ]: 3611 : || config->dry_run_with_checksums == true;
142 : :
143 : : /*
144 : : * Shared random-stop state for SHA512 test hooks.
145 : : * Test builds use these values to coordinate byte-exact interruption,
146 : : * byte-exact checkpointing, and the optional crash-after-checkpoint scenario
147 : : */
148 : : #ifdef TESTITALL_TEST_HOOKS
149 : : /*
150 : : * 0 means random-stop flow is disabled for this file.
151 : : * Non-zero means upper bound for random stop byte selection.
152 : : */
153 : 3611 : uint64_t random_stop_limit = 0U;
154 : : /* 0 means stop byte has not been selected yet. */
155 : 3611 : uint64_t random_stop_byte_value = 0U;
156 : : /* Separate state flag: do not overload stop-byte numeric value. */
157 : 3611 : bool random_stop_triggered = false;
158 : 3611 : bool testitall_checkpoint_at_random_byte = testitall_hash_checkpoint_at_random_byte_enabled();
159 : : #endif
160 : :
161 [ + + ]: 3611 : if(file->checksum_offset == 0)
162 : : {
163 : : // Fresh hashing pass: initialize the SHA512 state from scratch
164 [ - + ]: 3606 : if(sha512_init(&file->mdContext) != CRYPT_OK)
165 : : {
166 : 0 : slog(ERROR,"SHA512 initialization failed\n");
167 : 0 : fclose(fileptr);
168 : 0 : provide(FAILURE);
169 : : }
170 : : }
171 : :
172 : : /*
173 : : * Test hook A: choose the byte where the test should interfere.
174 : : * Later hook comments refer to hook A because signal waiting, read limiting,
175 : : * and byte-forced checkpoints all need the same exact target byte
176 : : */
177 : : #ifdef TESTITALL_TEST_HOOKS
178 : : /*
179 : : * Activate random interruption for a fresh pass of hugetestfile.
180 : : * Byte-checkpoint tests may also select a stop point after resume, so
181 : : * checkpoint-update paths can be exercised without a signal
182 : : */
183 [ + + + + ]: 3611 : if((file->checksum_offset == 0 || testitall_checkpoint_at_random_byte == true)
184 [ + + ]: 3607 : && testitall_is_huge_interruption_target(runtime_relative_path) == true
185 [ + - ]: 11 : && file->stat.st_size > 0)
186 : : {
187 : 11 : const uint64_t file_size = (uint64_t)file->stat.st_size;
188 : 11 : uint64_t current_offset = 0U;
189 : :
190 [ + + ]: 11 : if(file->checksum_offset > 0)
191 : : {
192 : 1 : current_offset = (uint64_t)file->checksum_offset;
193 : : }
194 : :
195 [ + - ]: 11 : if(current_offset < file_size)
196 : : {
197 : 11 : const uint64_t remaining_file_size = file_size - current_offset;
198 : :
199 [ + + - + ]: 11 : if(testitall_checkpoint_at_random_byte == true && remaining_file_size <= 1U)
200 : : {
201 : 0 : random_stop_limit = 0U;
202 : :
203 : : } else {
204 : 11 : random_stop_limit = file_size;
205 : :
206 : : /*
207 : : * Select interruption target before the first fread() call so
208 : : * the first chunk can be bounded and the stop point can land
209 : : * anywhere in the remaining file range.
210 : : */
211 : 11 : random_stop_byte_value = current_offset + testitall_random_stop_byte(remaining_file_size);
212 : :
213 : : /* Defensive fallback: never allow a zero stop byte. */
214 [ - + ]: 11 : if(random_stop_byte_value == 0U)
215 : : {
216 : 0 : random_stop_byte_value = 1U;
217 : : }
218 : :
219 : : /*
220 : : * Keep the first byte-checkpoint stop at least two bytes before EOF.
221 : : * This leaves room for the resume pass to write another checkpoint
222 : : * before the final hash state is reached
223 : : */
224 [ + + ]: 11 : if(testitall_checkpoint_at_random_byte == true
225 [ + + ]: 5 : && current_offset == 0U
226 [ + - ]: 4 : && random_stop_limit > 2U
227 [ - + ]: 4 : && random_stop_byte_value >= random_stop_limit - 1U)
228 : : {
229 : 0 : random_stop_byte_value = random_stop_limit - 2U;
230 : :
231 [ + - - + ]: 11 : } else if(random_stop_limit > 1U && random_stop_byte_value >= random_stop_limit){
232 : : /*
233 : : * Keep the stop point strictly inside file data for multi-byte files.
234 : : * If random selection lands exactly at EOF, shift it one byte left.
235 : : * The guard keeps subtraction safe and avoids unsigned underflow
236 : : */
237 : 0 : random_stop_byte_value = random_stop_limit - 1U;
238 : : }
239 : :
240 [ - + - - ]: 11 : if(random_stop_byte_value <= current_offset && current_offset + 1U < random_stop_limit)
241 : : {
242 : 0 : random_stop_byte_value = current_offset + 1U;
243 : : }
244 : : }
245 : : }
246 : : }
247 : : #endif
248 : :
249 [ + + ]: 3611 : if(perform_file_hashing == true)
250 : : {
251 : 2969 : long long int hashing_start_ns = cur_time_monotonic_ns();
252 : :
253 : : /*
254 : : * Monotonic timestamp when the next periodic checkpoint should be considered.
255 : : * It is advanced after each checkpoint window so the database is not updated on every read block
256 : : */
257 : 2969 : long long int next_checkpoint_ns = hashing_start_ns + DB_CHECKPOINT_INTERVAL_NS;
258 : :
259 : 2969 : unsigned char *file_buffer_data_rewritable = m_raw_data(file_buffer);
260 : :
261 : : while(true)
262 : 4534 : {
263 : : /*
264 : : * Test hook B: notify the signal-driven interruption test after the byte
265 : : * selected by hook A has been hashed. The delay hook uses this as the
266 : : * proof that the controlled stop point was really reached
267 : : */
268 : : #ifdef TESTITALL_TEST_HOOKS
269 : : /*
270 : : * Trigger point 2 exactly once when selected stop byte is reached.
271 : : */
272 [ + + ]: 7503 : if(random_stop_limit > 0U
273 [ + + ]: 1165 : && testitall_checkpoint_at_random_byte == false
274 [ + + ]: 710 : && random_stop_triggered == false
275 [ + - ]: 295 : && random_stop_byte_value > 0U
276 [ + + ]: 295 : && (uint64_t)(file->checksum_offset) >= random_stop_byte_value)
277 : : {
278 : 6 : signal_wait_at_point(2U);
279 : 6 : random_stop_triggered = true;
280 : : }
281 : : #endif
282 : :
283 : : /* Interrupt the loop smoothly */
284 : : /* Interrupt when Ctrl+C */
285 : : /*
286 : : * Test hook C: decide whether normal Ctrl+C/SIGINT handling must wait.
287 : : * This protects the signal-driven random-stop test until hook B has
288 : : * reached the byte selected by hook A
289 : : */
290 : : #ifdef TESTITALL_TEST_HOOKS
291 : : /*
292 : : * Test-only guard: when random-stop mode is active for hugetestfile,
293 : : * do not break on global_interrupt_flag until point 2 has really
294 : : * happened. Otherwise interruption may fire too early and miss the
295 : : * controlled "interrupt at random byte" scenario.
296 : : */
297 : 7503 : bool delay_interrupt_for_random_stop = false;
298 : :
299 [ + + ]: 7503 : if(random_stop_limit > 0U
300 [ + + ]: 1165 : && testitall_checkpoint_at_random_byte == false
301 [ + + ]: 710 : && random_stop_triggered == false)
302 : : {
303 [ - + ]: 289 : if(random_stop_byte_value == 0U)
304 : : {
305 : : /* No stop byte yet: wait until at least one block is hashed. */
306 : 0 : delay_interrupt_for_random_stop = true;
307 : :
308 [ + - ]: 289 : } else if((uint64_t)(file->checksum_offset) < random_stop_byte_value){
309 : : /* Stop byte is known but not reached yet: keep hashing. */
310 : 289 : delay_interrupt_for_random_stop = true;
311 : : }
312 : : }
313 : : #endif
314 : :
315 : : /*
316 : : * The normal interrupt condition also honors hook C in test builds.
317 : : * This keeps production behavior intact while letting the test reach
318 : : * the controlled stop byte before breaking the read loop
319 : : */
320 [ + + ]: 7503 : if(global_interrupt_flag == true
321 : : #ifdef TESTITALL_TEST_HOOKS
322 [ + - ]: 2 : && delay_interrupt_for_random_stop == false
323 : : #endif
324 : : )
325 : : {
326 : 2 : loop_was_interrupted = true;
327 : 2 : break;
328 : : }
329 : :
330 : 7501 : size_t read_limit = file_buffer->length;
331 : :
332 : : /*
333 : : * Cap one fread() call so the loop lands exactly on the byte selected
334 : : * by hook A. Without this cap, a large read could jump past the target
335 : : * and make the checkpoint/resume test flaky
336 : : */
337 : : #ifdef TESTITALL_TEST_HOOKS
338 : : /*
339 : : * Keep the read bounded so offset can stop exactly at the selected
340 : : * byte instead of jumping to EOF in a single large fread().
341 : : */
342 [ + + ]: 7501 : if(random_stop_limit > 0U
343 [ + + ]: 1163 : && random_stop_triggered == false
344 [ + - ]: 543 : && random_stop_byte_value > 0U
345 [ + - ]: 543 : && (uint64_t)(file->checksum_offset) < random_stop_byte_value)
346 : : {
347 : 543 : const uint64_t bytes_left_to_stop = random_stop_byte_value - (uint64_t)(file->checksum_offset);
348 : :
349 [ + + ]: 543 : if(bytes_left_to_stop < (uint64_t)read_limit)
350 : : {
351 : 11 : read_limit = (size_t)bytes_left_to_stop;
352 : : }
353 : : }
354 : : #endif
355 : :
356 : 7501 : size_t len = fread(file_buffer_data_rewritable,sizeof(unsigned char),read_limit,fileptr);
357 : :
358 [ + + ]: 7501 : if(len == 0)
359 : : {
360 [ + + ]: 2964 : if(ferror(fileptr))
361 : : {
362 : 1 : file->read_error = true;
363 : 1 : file->read_errno = errno;
364 : : }
365 : :
366 : 2964 : break;
367 : : }
368 : :
369 [ + - ]: 4537 : if(SUCCESS == status)
370 : : {
371 [ - + ]: 4537 : if(sha512_update(&file->mdContext,file_buffer_data_rewritable,len) != CRYPT_OK)
372 : : {
373 : 0 : slog(ERROR,"SHA512 update failed\n");
374 : 0 : status = FAILURE;
375 : 0 : break;
376 : : }
377 : :
378 : 4537 : file->checksum_offset += (sqlite3_int64)len;
379 : 4537 : summary->total_hashed_bytes += len;
380 : :
381 : : /*
382 : : * Test hook D: request an immediate checkpoint when the byte selected
383 : : * by hook A has just been hashed. The checkpoint condition and crash
384 : : * simulation use this flag to avoid waiting for the time interval
385 : : */
386 : : #ifdef TESTITALL_TEST_HOOKS
387 : 4537 : bool testitall_checkpoint_now = false;
388 : :
389 [ + + ]: 4537 : if(testitall_checkpoint_at_random_byte == true
390 [ + - ]: 453 : && random_stop_limit > 0U
391 [ + + ]: 453 : && random_stop_triggered == false
392 [ + - ]: 254 : && random_stop_byte_value > 0U
393 [ + + ]: 254 : && (uint64_t)(file->checksum_offset) >= random_stop_byte_value)
394 : : {
395 : 5 : testitall_checkpoint_now = true;
396 : 5 : random_stop_triggered = true;
397 : : }
398 : : #endif
399 : :
400 : : /*
401 : : * Current monotonic timestamp sampled after the SHA512 state and byte offset were advanced.
402 : : * Saving only after this point keeps offset and mdContext consistent in the database
403 : : */
404 : 4537 : const long long int checkpoint_now_ns = cur_time_monotonic_ns();
405 : :
406 : : /*
407 : : * Honor hook D by entering the checkpoint path immediately.
408 : : * This is what makes byte-precise checkpoint tests practical even
409 : : * when the normal time-based checkpoint interval has not elapsed
410 : : */
411 [ + - ]: 4537 : if(checkpoint_now_ns >= next_checkpoint_ns
412 [ + + ]: 4537 : || (config->file_transaction_active == true
413 [ + - ]: 2582 : && checkpoint_now_ns - config->file_transaction_started_ns >= DB_CHECKPOINT_INTERVAL_NS)
414 : : #ifdef TESTITALL_TEST_HOOKS
415 [ + + ]: 4537 : || testitall_checkpoint_now == true
416 : : #endif
417 : : )
418 : : {
419 : : /*
420 : : * Test hook E: remember whether this checkpoint really reached
421 : : * the database. The crash simulation relies on hook E so it only
422 : : * exits after there is durable state to resume from
423 : : */
424 : : #ifdef TESTITALL_TEST_HOOKS
425 : 5 : bool checkpoint_saved = false;
426 : : #endif
427 : :
428 : 5 : const bool checkpoint_allowed = periodic_hash_checkpoint_is_allowed(file);
429 : :
430 [ + + ]: 5 : if(checkpoint_allowed == true)
431 : : {
432 : 4 : status = db_save_file_record(relative_path,file,path_known,false);
433 : :
434 [ - + ]: 4 : if(FAILURE & status)
435 : : {
436 : 0 : break;
437 : : }
438 : : }
439 : :
440 : 5 : call(db_file_transaction_commit());
441 : :
442 [ - + ]: 5 : if(FAILURE & status)
443 : : {
444 : 0 : break;
445 : : }
446 : :
447 : : /*
448 : : * Set hook E only after the checkpoint save and COMMIT succeed.
449 : : * This connects the durable DB write above with the crash
450 : : * simulation below
451 : : */
452 : : #ifdef TESTITALL_TEST_HOOKS
453 : 5 : checkpoint_saved = checkpoint_allowed;
454 : : #endif
455 : :
456 : : /*
457 : : * Simulate a sudden process death only when hook D requested this
458 : : * byte checkpoint and hook E proves it was saved. The next test run
459 : : * can then prove that the stored partial SHA512 state survived
460 : : */
461 : : #ifdef TESTITALL_TEST_HOOKS
462 [ + + ]: 5 : if(checkpoint_saved == true
463 [ + - ]: 4 : && testitall_checkpoint_now == true
464 [ + + ]: 4 : && testitall_exit_after_hash_checkpoint_enabled() == true)
465 : : {
466 : 3 : testitall_exit_after_hash_checkpoint();
467 : : }
468 : : #endif
469 : :
470 : 2 : next_checkpoint_ns = checkpoint_now_ns + DB_CHECKPOINT_INTERVAL_NS;
471 : : }
472 : : }
473 : : }
474 : :
475 : 2966 : long long int hashing_stop_ns = cur_time_monotonic_ns();
476 : :
477 : 2966 : long long int hashing_elapsed_ns = hashing_stop_ns - hashing_start_ns;
478 : :
479 [ - + ]: 2966 : if(hashing_elapsed_ns < 0LL)
480 : : {
481 : 0 : hashing_elapsed_ns = 0LL;
482 : : }
483 : :
484 : 2966 : summary->total_hashing_elapsed_ns += hashing_elapsed_ns;
485 : : }
486 : :
487 [ - + ]: 3608 : if(fclose(fileptr) != 0)
488 : : {
489 : 0 : slog(ERROR,"Error closing file %s\n",runtime_relative_path);
490 : : }
491 : :
492 [ + - ]: 3608 : if(SUCCESS == status
493 [ + + ]: 3608 : && perform_file_hashing == true
494 [ + + ]: 2966 : && loop_was_interrupted == false)
495 : : {
496 : 2964 : file->checksum_offset = 0;
497 : :
498 [ - + ]: 2964 : if(sha512_final(&file->mdContext,file->sha512) != CRYPT_OK)
499 : : {
500 : 0 : slog(ERROR,"SHA512 finalization failed\n");
501 : 0 : status = FAILURE;
502 : : }
503 : : }
504 : :
505 : 3608 : provide(status);
506 : : }
|
