From 7ab14b626e870185251a19d28031e60f237d92cb Mon Sep 17 00:00:00 2001 From: Magnus Lundborg Date: Mon, 16 Jun 2014 08:06:21 +0200 Subject: Migrating data blocks [WIP]. Also includes some simple tests to verify that it is working. Change-Id: I6d465000a6fd202d53af033d7e839bfecdccc068 diff --git a/include/tng_io.h b/include/tng_io.h index ebdc618..4ca5027 100644 --- a/include/tng_io.h +++ b/include/tng_io.h @@ -413,6 +413,7 @@ typedef enum {TNG_NON_TRAJECTORY_BLOCK, TNG_TRAJECTORY_BLOCK} tng_block_type; #define TNG_TRAJ_B_FACTORS 0x0000000010000006LL #define TNG_TRAJ_ANISOTROPIC_B_FACTORS 0x0000000010000007LL #define TNG_TRAJ_OCCUPANCY 0x0000000010000008LL +#define TNG_TRAJ_GENERAL_COMMENTS 0x0000000010000009LL /** @} */ diff --git a/src/lib/tng_io.c b/src/lib/tng_io.c index 83ea162..2dd48f4 100644 --- a/src/lib/tng_io.c +++ b/src/lib/tng_io.c @@ -1,7 +1,7 @@ /* This code is part of the tng binary trajectory format. * * Written by Magnus Lundborg - * Copyright (c) 2012-2013, The GROMACS development team. + * Copyright (c) 2012-2014, The GROMACS development team. * Check out http://www.gromacs.org for more information. * * @@ -1027,302 +1027,664 @@ static tng_function_status tng_block_header_read // } */ -static tng_function_status tng_reread_frame_set_at_file_pos - (tng_trajectory_t tng_data, - const int64_t pos) +/** Update the md5 hash of a block already written to the file + * @param tng_data is a trajectory data container. + * @param block is the block, of which to update the md5 hash. + * @param header_start_pos is the file position where the block header starts. + * @param contents_start_pos is the file position where the block contents + * start. + * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major + * error has occured. + */ +static tng_function_status tng_md5_hash_update(tng_trajectory_t tng_data, + tng_gen_block_t block, + const int64_t header_start_pos, + const int64_t contents_start_pos) { - tng_gen_block_t block; - tng_function_status stat; - - tng_block_init(&block); + if(block->block_contents) + { + free(block->block_contents); + } - fseek(tng_data->input_file, pos, SEEK_SET); - if(pos > 0) + block->block_contents = malloc(block->block_contents_size); + if(!block->block_contents) { - stat = tng_block_header_read(tng_data, block); - if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) - { - fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", pos, - __FILE__, __LINE__); - tng_block_destroy(&block); - return(TNG_FAILURE); - } + fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", + block->block_contents_size, __FILE__, __LINE__); + return(TNG_CRITICAL); + } - if(tng_block_read_next(tng_data, block, - TNG_SKIP_HASH) != TNG_SUCCESS) - { - tng_block_destroy(&block); - return(TNG_CRITICAL); - } + fseek(tng_data->output_file, (long)contents_start_pos, SEEK_SET); + if(fread(block->block_contents, block->block_contents_size, 1, + tng_data->output_file) == 0) + { + fprintf(stderr, "TNG library: Cannot read block. %s: %d\n", __FILE__, __LINE__); + return(TNG_CRITICAL); } - tng_block_destroy(&block); + tng_block_md5_hash_generate(block); + + fseek(tng_data->output_file, (long)header_start_pos + 3 * sizeof(int64_t), + SEEK_SET); + fwrite(block->md5_hash, TNG_MD5_HASH_LEN, 1, tng_data->output_file); return(TNG_SUCCESS); } -static tng_function_status tng_file_pos_of_subsequent_trajectory_block_get - (tng_trajectory_t tng_data, - int64_t *pos) +/** Update the frame set pointers in the file header (general info block), + * already written to disk + * @param tng_data is a trajectory data container. + * @param hash_mode specifies whether to update the block md5 hash when + * updating the pointers. + * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major + * error has occured. + */ +static tng_function_status tng_header_pointers_update + (tng_trajectory_t tng_data, const char hash_mode) { - int64_t orig_pos, curr_frame_set_pos; tng_gen_block_t block; - tng_function_status stat; - tng_trajectory_frame_set_t frame_set = - &tng_data->current_trajectory_frame_set; - - orig_pos = ftell(tng_data->input_file); - curr_frame_set_pos = tng_data->current_trajectory_frame_set_input_file_pos; - - *pos = tng_data->first_trajectory_frame_set_input_file_pos; + FILE *temp = tng_data->input_file; + int64_t output_file_pos, pos, contents_start_pos; - if(*pos <= 0) + if(tng_output_file_init(tng_data) != TNG_SUCCESS) { - return(TNG_SUCCESS); + fprintf(stderr, "TNG library: Cannot initialise destination file. %s: %d\n", + __FILE__, __LINE__); + return(TNG_CRITICAL); } - fseek(tng_data->input_file, *pos, SEEK_SET); + tng_data->input_file = tng_data->output_file; tng_block_init(&block); - /* Read block headers first to see that a frame set block is found. */ - stat = tng_block_header_read(tng_data, block); - if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) - { - fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", *pos, - __FILE__, __LINE__); - tng_block_destroy(&block); - return(TNG_FAILURE); - } - if(tng_block_read_next(tng_data, block, - TNG_SKIP_HASH) != TNG_SUCCESS) + output_file_pos = ftell(tng_data->output_file); + fseek(tng_data->output_file, 0, SEEK_SET); + + if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) { + fprintf(stderr, "TNG library: Cannot read general info header. %s: %d\n", + __FILE__, __LINE__); + tng_data->input_file = temp; tng_block_destroy(&block); return(TNG_CRITICAL); } - /* Read all frame set blocks (not the blocks between them) */ - while(frame_set->next_frame_set_file_pos > 0) - { - fseek(tng_data->input_file, frame_set->next_frame_set_file_pos, SEEK_SET); - stat = tng_block_header_read(tng_data, block); - if(stat == TNG_CRITICAL) - { - fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", *pos, - __FILE__, __LINE__); - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - if(stat != TNG_SUCCESS || block->id != TNG_TRAJECTORY_FRAME_SET) - { - return(TNG_FAILURE); - } - - stat = tng_block_read_next(tng_data, block, TNG_SKIP_HASH); - if(stat != TNG_SUCCESS) - { - tng_block_destroy(&block); - return(stat); - } - /* Update *pos if this is the earliest frame set so far (after orig_pos) */ - if(tng_data->current_trajectory_frame_set_input_file_pos < *pos && - tng_data->current_trajectory_frame_set_input_file_pos > orig_pos) - { - *pos = tng_data->current_trajectory_frame_set_input_file_pos; - } - } - - /* Re-read the frame set that used to be the current one */ - tng_reread_frame_set_at_file_pos(tng_data, curr_frame_set_pos); - - fseek(tng_data->input_file, orig_pos, SEEK_SET); - - tng_block_destroy(&block); + contents_start_pos = ftell(tng_data->output_file); - return(TNG_SUCCESS); -} + fseek(tng_data->output_file, (long)block->block_contents_size - 5 * + sizeof(int64_t), SEEK_CUR); -static tng_function_status tng_file_part_migrate(tng_trajectory_t tng_data, - int64_t block_start_pos, - int64_t block_len, - int64_t new_pos) -{ - int64_t i; + tng_data->input_file = temp; - char *contents; + pos = tng_data->first_trajectory_frame_set_output_file_pos; - if(tng_input_file_init(tng_data) != TNG_SUCCESS) + if(tng_data->input_endianness_swap_func_64) { - return(TNG_CRITICAL); + if(tng_data->input_endianness_swap_func_64(tng_data, + &pos) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } } - contents = malloc(block_len); - if(!contents) + if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) { - fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", - block_len, __FILE__, __LINE__); + tng_block_destroy(&block); return(TNG_CRITICAL); } - /* First read the header size to be able to read the whole header. */ - if(fread(contents, block_len, 1, tng_data->input_file) == 0) + pos = tng_data->last_trajectory_frame_set_output_file_pos; + + if(tng_data->input_endianness_swap_func_64) { - fprintf(stderr, "TNG library: Cannot read data from file when migrating data. %s: %d\n", - __FILE__, __LINE__); - free(contents); - return(TNG_CRITICAL); + if(tng_data->input_endianness_swap_func_64(tng_data, + &pos) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } } - fseek(tng_data->output_file, new_pos, SEEK_SET); - if(fwrite(contents, block_len, 1, tng_data->output_file) != 1) + if(fwrite(&pos, + sizeof(int64_t), 1, tng_data->output_file) != 1) { - fprintf(stderr, "TNG library: Could not write data to file when migrating data. %s: %d\n", - __FILE__, __LINE__); - free(contents); + tng_block_destroy(&block); return(TNG_CRITICAL); } - fseek(tng_data->output_file, new_pos, SEEK_SET); - - /* Fill the block with NULL to avoid confusion. */ - for(i = 0; i < block_len; i++) + if(hash_mode == TNG_USE_HASH) { - contents[i] = '\0'; + tng_md5_hash_update(tng_data, block, 0, contents_start_pos); } - /* FIXME: casting block_len to size_t is dangerous */ - fwrite(contents, 1, block_len, tng_data->output_file); - free(contents); + tng_block_destroy(&block); + + fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); return(TNG_SUCCESS); } -static tng_function_status tng_length_of_current_frame_set_contents_get - (tng_trajectory_t tng_data, - int64_t *len) +/** Update the frame set pointers in the current frame set block, already + * written to disk. It also updates the pointers of the blocks pointing to + * the current frame set block. + * @param tng_data is a trajectory data container. + * @param hash_mode specifies whether to update the block md5 hash when + * updating the pointers. + * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major + * error has occured. + */ +static tng_function_status tng_frame_set_pointers_update + (tng_trajectory_t tng_data, const char hash_mode) { - int64_t orig_pos, curr_frame_set_pos; tng_gen_block_t block; - tng_function_status stat; + tng_trajectory_frame_set_t frame_set; + FILE *temp = tng_data->input_file; + int64_t pos, output_file_pos, header_start_pos, contents_start_pos; - orig_pos = ftell(tng_data->input_file); - curr_frame_set_pos = tng_data->current_trajectory_frame_set_input_file_pos; + if(tng_output_file_init(tng_data) != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot initialise destination file. %s: %d\n", + __FILE__, __LINE__); + return(TNG_CRITICAL); + } - *len = 0; + tng_block_init(&block); + output_file_pos = ftell(tng_data->output_file); - fseek(tng_data->input_file, curr_frame_set_pos, SEEK_SET); + tng_data->input_file = tng_data->output_file; - tng_block_init(&block); - /* Read block headers first to see that a frame set block is found. */ - stat = tng_block_header_read(tng_data, block); - if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) - { - fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", - curr_frame_set_pos, __FILE__, __LINE__); - tng_block_destroy(&block); - return(TNG_FAILURE); - } + frame_set = &tng_data->current_trajectory_frame_set; - /* Read the headers of all blocks in the frame set (not the actual contents of them) */ - while(stat == TNG_SUCCESS) + /* Update previous frame set */ + if(frame_set->prev_frame_set_file_pos != -1 && + frame_set->prev_frame_set_file_pos != 0) { - *len += block->header_contents_size; - fseek(tng_data->input_file, block->block_contents_size, SEEK_CUR); - *len += block->block_contents_size; - stat = tng_block_header_read(tng_data, block); - if(block->id == TNG_TRAJECTORY_FRAME_SET) + fseek(tng_data->output_file, (long)frame_set->prev_frame_set_file_pos, + SEEK_SET); + + header_start_pos = frame_set->prev_frame_set_file_pos; + + if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) { + fprintf(stderr, "TNG library: Cannot read frame header. %s: %d\n", + __FILE__, __LINE__); + tng_data->input_file = temp; tng_block_destroy(&block); - return(TNG_SUCCESS); + return(TNG_CRITICAL); + } + + contents_start_pos = ftell(tng_data->output_file); + + fseek(tng_data->output_file, (long)block->block_contents_size - (6 * + sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); + + pos = tng_data->current_trajectory_frame_set_output_file_pos; + + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + &pos) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } } + + if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_CRITICAL); + } + + if(hash_mode == TNG_USE_HASH) + { + tng_md5_hash_update(tng_data, block, header_start_pos, + contents_start_pos); + } + fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); } - /* Re-read the frame set that used to be the current one */ - tng_reread_frame_set_at_file_pos(tng_data, curr_frame_set_pos); + /* Update the frame set one medium stride step before */ + if(frame_set->medium_stride_prev_frame_set_file_pos != -1 && + frame_set->medium_stride_prev_frame_set_file_pos != 0) + { + fseek(tng_data->output_file, + (long)frame_set->medium_stride_prev_frame_set_file_pos, + SEEK_SET); - fseek(tng_data->input_file, orig_pos, SEEK_SET); + if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot read frame set header. %s: %d\n", + __FILE__, __LINE__); + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_CRITICAL); + } + + contents_start_pos = ftell(tng_data->output_file); + + fseek(tng_data->output_file, (long)block->block_contents_size - (4 * + sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); + + pos = tng_data->current_trajectory_frame_set_output_file_pos; + + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + &pos) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } + } + + if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_CRITICAL); + } + + if(hash_mode == TNG_USE_HASH) + { + tng_md5_hash_update(tng_data, block, + frame_set->medium_stride_prev_frame_set_file_pos, + contents_start_pos); + } + } + + /* Update the frame set one long stride step before */ + if(frame_set->long_stride_prev_frame_set_file_pos != -1 && + frame_set->long_stride_prev_frame_set_file_pos != 0) + { + fseek(tng_data->output_file, + (long)frame_set->long_stride_prev_frame_set_file_pos, + SEEK_SET); + + if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot read frame set header. %s: %d\n", + __FILE__, __LINE__); + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_CRITICAL); + } + + contents_start_pos = ftell(tng_data->output_file); + + fseek(tng_data->output_file, (long)block->block_contents_size - (2 * + sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); + + pos = tng_data->current_trajectory_frame_set_output_file_pos; + + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + &pos) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } + } + + if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_CRITICAL); + } + + if(hash_mode == TNG_USE_HASH) + { + tng_md5_hash_update(tng_data, block, + frame_set->long_stride_prev_frame_set_file_pos, + contents_start_pos); + } + } + + fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); + + tng_data->input_file = temp; tng_block_destroy(&block); return(TNG_SUCCESS); } -/** Migrate blocks in the file to make room for new data in a block. This - * is required e.g. when adding data to a block or extending strings in a - * block. - * @param tng_data is a trajectory data container. - * @param start_pos is the position from which to start moving data, usually - * the byte after the end of the block to which data was added. - * @param offset is the number of bytes that were inserted. - * @details Trajectory blocks (frame sets and their related blocks) are moved - * to the end of the file (if needed) in order to make room for non-trajectory - * data. - */ -static tng_function_status tng_migrate_data_in_file +static tng_function_status tng_reread_frame_set_at_file_pos (tng_trajectory_t tng_data, - int64_t start_pos, - int64_t offset) + const int64_t pos) { - int64_t traj_start_pos, empty_space, orig_file_pos, frame_set_length; tng_gen_block_t block; tng_function_status stat; - FILE *temp; - if(offset <= 0) + tng_block_init(&block); + + fseek(tng_data->input_file, pos, SEEK_SET); + if(pos > 0) { - return(TNG_SUCCESS); + stat = tng_block_header_read(tng_data, block); + if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) + { + fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", pos, + __FILE__, __LINE__); + tng_block_destroy(&block); + return(TNG_FAILURE); + } + + if(tng_block_read_next(tng_data, block, + TNG_SKIP_HASH) != TNG_SUCCESS) + { + tng_block_destroy(&block); + return(TNG_CRITICAL); + } } - temp = tng_data->input_file; + tng_block_destroy(&block); - stat = tng_file_pos_of_subsequent_trajectory_block_get(tng_data, &traj_start_pos); - if(stat != TNG_SUCCESS) + return(TNG_SUCCESS); +} + +static tng_function_status tng_file_pos_of_subsequent_trajectory_block_get + (tng_trajectory_t tng_data, + int64_t *pos) +{ + int64_t orig_pos, curr_frame_set_pos; + tng_gen_block_t block; + tng_function_status stat; + tng_trajectory_frame_set_t frame_set = + &tng_data->current_trajectory_frame_set; + + orig_pos = ftell(tng_data->input_file); + curr_frame_set_pos = tng_data->current_trajectory_frame_set_input_file_pos; + + *pos = tng_data->first_trajectory_frame_set_input_file_pos; + + if(*pos <= 0) { - tng_data->input_file = temp; - return(stat); + return(TNG_SUCCESS); } - empty_space = traj_start_pos - start_pos - 1; + fseek(tng_data->input_file, *pos, SEEK_SET); - if(empty_space >= offset) + tng_block_init(&block); + /* Read block headers first to see that a frame set block is found. */ + stat = tng_block_header_read(tng_data, block); + if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) { - return(TNG_SUCCESS); + fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", *pos, + __FILE__, __LINE__); + tng_block_destroy(&block); + return(TNG_FAILURE); } - orig_file_pos = ftell(tng_data->input_file); + if(tng_block_read_next(tng_data, block, + TNG_SKIP_HASH) != TNG_SUCCESS) + { + tng_block_destroy(&block); + return(TNG_CRITICAL); + } - while(empty_space < offset) + /* Read all frame set blocks (not the blocks between them) */ + while(frame_set->next_frame_set_file_pos > 0) { - fseek(tng_data->input_file, traj_start_pos, SEEK_SET); + fseek(tng_data->input_file, frame_set->next_frame_set_file_pos, SEEK_SET); stat = tng_block_header_read(tng_data, block); if(stat == TNG_CRITICAL) { - fprintf(stderr, "TNG library: Cannot read block header. %s: %d\n", + fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", *pos, __FILE__, __LINE__); tng_block_destroy(&block); - tng_data->input_file = temp; return(TNG_CRITICAL); } if(stat != TNG_SUCCESS || block->id != TNG_TRAJECTORY_FRAME_SET) { - tng_data->input_file = temp; return(TNG_FAILURE); } - stat = tng_length_of_current_frame_set_contents_get(tng_data, &frame_set_length); + + stat = tng_block_read_next(tng_data, block, TNG_SKIP_HASH); if(stat != TNG_SUCCESS) { - tng_data->input_file = temp; + tng_block_destroy(&block); return(stat); } - stat = tng_file_part_migrate(tng_data, traj_start_pos, - frame_set_length, tng_data->input_file_len); - if(stat != TNG_SUCCESS) + /* Update *pos if this is the earliest frame set so far (after orig_pos) */ + if(tng_data->current_trajectory_frame_set_input_file_pos < *pos && + tng_data->current_trajectory_frame_set_input_file_pos > orig_pos) { - tng_data->input_file = temp; - return(stat); + *pos = tng_data->current_trajectory_frame_set_input_file_pos; } } - fseek(tng_data->input_file, orig_file_pos, SEEK_SET); + + /* Re-read the frame set that used to be the current one */ + tng_reread_frame_set_at_file_pos(tng_data, curr_frame_set_pos); + + fseek(tng_data->input_file, orig_pos, SEEK_SET); + + tng_block_destroy(&block); + + return(TNG_SUCCESS); +} + +static tng_function_status tng_frame_set_complete_migrate + (tng_trajectory_t tng_data, + int64_t block_start_pos, + int64_t block_len, + int64_t new_pos) +{ + int64_t i; + tng_bool updated = TNG_FALSE; + + char *contents; + + if(tng_input_file_init(tng_data) != TNG_SUCCESS) + { + return(TNG_CRITICAL); + } + + contents = malloc(block_len); + if(!contents) + { + fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", + block_len, __FILE__, __LINE__); + return(TNG_CRITICAL); + } + + if(fread(contents, block_len, 1, tng_data->input_file) == 0) + { + fprintf(stderr, "TNG library: Cannot read data from file when migrating data. %s: %d\n", + __FILE__, __LINE__); + free(contents); + return(TNG_CRITICAL); + } + fseek(tng_data->output_file, new_pos, SEEK_SET); + + if(fwrite(contents, block_len, 1, tng_data->output_file) != 1) + { + fprintf(stderr, "TNG library: Could not write data to file when migrating data. %s: %d\n", + __FILE__, __LINE__); + free(contents); + return(TNG_CRITICAL); + } + + tng_frame_set_pointers_update(tng_data, TNG_USE_HASH); + + /* Update the general info block if needed */ + if(block_start_pos == tng_data->first_trajectory_frame_set_output_file_pos) + { + tng_data->first_trajectory_frame_set_output_file_pos = new_pos; + updated = TNG_TRUE; + } + if(block_start_pos == tng_data->last_trajectory_frame_set_output_file_pos) + { + tng_data->last_trajectory_frame_set_output_file_pos = new_pos; + updated = TNG_TRUE; + } + if(updated) + { + tng_header_pointers_update(tng_data, TNG_USE_HASH); + } + + /* Fill the block with NULL to avoid confusion. */ + for(i = 0; i < block_len; i++) + { + contents[i] = '\0'; + } + fseek(tng_data->output_file, block_start_pos, SEEK_SET); + + /* FIXME: casting block_len to size_t is dangerous */ + fwrite(contents, 1, block_len, tng_data->output_file); + + free(contents); + + return(TNG_SUCCESS); +} + +static tng_function_status tng_length_of_current_frame_set_contents_get + (tng_trajectory_t tng_data, + int64_t *len) +{ + int64_t orig_pos, pos, curr_frame_set_pos; + tng_gen_block_t block; + tng_function_status stat; + + orig_pos = ftell(tng_data->input_file); + curr_frame_set_pos = pos = tng_data->current_trajectory_frame_set_input_file_pos; + + *len = 0; + + fseek(tng_data->input_file, curr_frame_set_pos, SEEK_SET); + + tng_block_init(&block); + /* Read block headers first to see that a frame set block is found. */ + stat = tng_block_header_read(tng_data, block); + if(stat == TNG_CRITICAL || block->id != TNG_TRAJECTORY_FRAME_SET) + { + fprintf(stderr, "TNG library: Cannot read block header at pos %"PRId64". %s: %d\n", + curr_frame_set_pos, __FILE__, __LINE__); + tng_block_destroy(&block); + return(TNG_FAILURE); + } + + /* Read the headers of all blocks in the frame set (not the actual contents of them) */ + while(stat == TNG_SUCCESS) + { + fseek(tng_data->input_file, block->block_contents_size, SEEK_CUR); + *len += block->header_contents_size + block->block_contents_size; + pos += block->header_contents_size + block->block_contents_size; + if(pos >= tng_data->input_file_len) + { + break; + } + stat = tng_block_header_read(tng_data, block); + if(block->id == TNG_TRAJECTORY_FRAME_SET) + { + break; + } + } + + /* Re-read the frame set that used to be the current one */ + tng_reread_frame_set_at_file_pos(tng_data, curr_frame_set_pos); + + fseek(tng_data->input_file, orig_pos, SEEK_SET); + + tng_block_destroy(&block); + + return(TNG_SUCCESS); +} + +/** Migrate blocks in the file to make room for new data in a block. This + * is required e.g. when adding data to a block or extending strings in a + * block. + * @param tng_data is a trajectory data container. + * @param start_pos is the position from which to start moving data, usually + * the byte after the end of the block to which data was added. + * @param offset is the number of bytes that were inserted. + * @details Trajectory blocks (frame sets and their related blocks) are moved + * to the end of the file (if needed) in order to make room for non-trajectory + * data. + */ +static tng_function_status tng_migrate_data_in_file + (tng_trajectory_t tng_data, + int64_t start_pos, + int64_t offset) +{ + int64_t traj_start_pos, empty_space, orig_file_pos, frame_set_length; + tng_gen_block_t block; + tng_function_status stat; + FILE *temp; + + if(offset <= 0) + { + return(TNG_SUCCESS); + } + + temp = tng_data->input_file; + + stat = tng_file_pos_of_subsequent_trajectory_block_get(tng_data, &traj_start_pos); + if(stat != TNG_SUCCESS) + { + tng_data->input_file = temp; + return(stat); + } + + tng_data->current_trajectory_frame_set_input_file_pos = traj_start_pos; + + empty_space = traj_start_pos - (start_pos - 1); + + if(empty_space >= offset) + { + return(TNG_SUCCESS); + } + + orig_file_pos = ftell(tng_data->input_file); + tng_block_init(&block); + + while(empty_space < offset) + { + fseek(tng_data->input_file, traj_start_pos, SEEK_SET); + stat = tng_block_header_read(tng_data, block); + if(stat == TNG_CRITICAL) + { + fprintf(stderr, "TNG library: Cannot read block header. %s: %d\n", + __FILE__, __LINE__); + tng_block_destroy(&block); + tng_data->input_file = temp; + return(TNG_CRITICAL); + } + if(stat != TNG_SUCCESS || block->id != TNG_TRAJECTORY_FRAME_SET) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(TNG_FAILURE); + } + stat = tng_length_of_current_frame_set_contents_get(tng_data, &frame_set_length); + if(stat != TNG_SUCCESS) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(stat); + } + stat = tng_frame_set_complete_migrate(tng_data, traj_start_pos, + frame_set_length, tng_data->input_file_len); + if(stat != TNG_SUCCESS) + { + tng_data->input_file = temp; + tng_block_destroy(&block); + return(stat); + } + + empty_space += frame_set_length; + } + fseek(tng_data->input_file, orig_file_pos, SEEK_SET); + tng_block_destroy(&block); return(TNG_SUCCESS); } @@ -2084,8 +2446,8 @@ static tng_function_status tng_general_info_block_write offset += sizeof(tng_data->frame_set_n_frames); memcpy(block->block_contents+offset, - &tng_data->first_trajectory_frame_set_input_file_pos, - sizeof(tng_data->first_trajectory_frame_set_input_file_pos)); + &tng_data->first_trajectory_frame_set_output_file_pos, + sizeof(tng_data->first_trajectory_frame_set_output_file_pos)); if(tng_data->output_endianness_swap_func_64) { if(tng_data->output_endianness_swap_func_64(tng_data, @@ -2096,11 +2458,11 @@ static tng_function_status tng_general_info_block_write __FILE__, __LINE__); } } - offset += sizeof(tng_data->first_trajectory_frame_set_input_file_pos); + offset += sizeof(tng_data->first_trajectory_frame_set_output_file_pos); memcpy(block->block_contents+offset, - &tng_data->last_trajectory_frame_set_input_file_pos, - sizeof(tng_data->last_trajectory_frame_set_input_file_pos)); + &tng_data->last_trajectory_frame_set_output_file_pos, + sizeof(tng_data->last_trajectory_frame_set_output_file_pos)); if(tng_data->output_endianness_swap_func_64) { if(tng_data->output_endianness_swap_func_64(tng_data, @@ -2111,7 +2473,7 @@ static tng_function_status tng_general_info_block_write __FILE__, __LINE__); } } - offset += sizeof(tng_data->last_trajectory_frame_set_input_file_pos); + offset += sizeof(tng_data->last_trajectory_frame_set_output_file_pos); memcpy(block->block_contents+offset, &tng_data->medium_stride_length, sizeof(tng_data->medium_stride_length)); @@ -2594,6 +2956,11 @@ static tng_function_status tng_molecules_block_len_calculate sizeof(molecule->n_bonds)) * tng_data->n_molecules; + if(!tng_data->var_num_atoms_flag) + { + *len += tng_data->n_molecules * sizeof(int64_t); + } + return(TNG_SUCCESS); } @@ -3092,11 +3459,6 @@ static tng_function_status tng_molecules_block_write return(TNG_CRITICAL); } - if(!tng_data->var_num_atoms_flag) - { - block->block_contents_size += tng_data->n_molecules * sizeof(int64_t); - } - block->block_contents = malloc(block->block_contents_size); if(!block->block_contents) { @@ -6649,670 +7011,340 @@ static tng_function_status tng_data_block_write(tng_trajectory_t tng_data, /* fprintf(stderr, "TNG library: Before compression: %"PRId64"\n", block->block_contents_size); */ stat = tng_gzip_compress(tng_data, block, block->block_contents + data_start_pos, - block->block_contents_size - data_start_pos); - if(stat != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Could not write gzipped block data. %s: %d\n", __FILE__, - __LINE__); - if(stat == TNG_CRITICAL) - { - return(TNG_CRITICAL); - } - data->codec_id = TNG_UNCOMPRESSED; - } - /* fprintf(stderr, "TNG library: After compression: %"PRId64"\n", block->block_contents_size); */ - break; -#endif - } - } - - if(tng_block_header_write(tng_data, block, hash_mode) != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot write header of file %s. %s: %d\n", - tng_data->output_file_path, __FILE__, __LINE__); - return(TNG_CRITICAL); - } - - if(fwrite(block->block_contents, block->block_contents_size, 1, - tng_data->output_file) != 1) - { - fprintf(stderr, "TNG library: Could not write all block data. %s: %d\n", - __FILE__, __LINE__); - return(TNG_CRITICAL); - } - - return(TNG_SUCCESS); -} - -/** Read the meta information of a data block (particle or non-particle data). - * @param tng_data is a trajectory data container. - * @param block is the block to store the data (should already contain - * the block headers). - * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major - * error has occured. - */ -static tng_function_status tng_data_block_meta_information_read - (tng_trajectory_t tng_data, - tng_gen_block_t block, - int *offset, - char *datatype, - char *dependency, - char *sparse_data, - int64_t *n_values, - int64_t *codec_id, - int64_t *first_frame_with_data, - int64_t *stride_length, - int64_t *n_frames, - int64_t *num_first_particle, - int64_t *block_n_particles, - double *multiplier) -{ - int meta_size; - char *contents; - - if(block->block_contents) - { - contents = block->block_contents; - } - else - { - meta_size = 3 * sizeof(char) + sizeof(double) + 6 * sizeof(int64_t); - contents = malloc(meta_size); - if(!contents) - { - fprintf(stderr, "TNG library: Cannot allocate memory (%d bytes). %s: %d\n", - meta_size, __FILE__, __LINE__); - } - - if(fread(contents, meta_size, 1, tng_data->input_file) == 0) - { - fprintf(stderr, "TNG library: Cannot read data block meta information. %s: %d\n", __FILE__, __LINE__); - free(contents); - return(TNG_CRITICAL); - } - } - - memcpy(datatype, contents+*offset, - sizeof(*datatype)); - *offset += sizeof(*datatype); - - memcpy(dependency, contents+*offset, - sizeof(*dependency)); - *offset += sizeof(*dependency); - - if(*dependency & TNG_FRAME_DEPENDENT) - { - memcpy(sparse_data, contents+*offset, - sizeof(*sparse_data)); - *offset += sizeof(*sparse_data); - } - - memcpy(n_values, contents+*offset, - sizeof(*n_values)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - n_values) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*n_values); - - memcpy(codec_id, contents+*offset, - sizeof(*codec_id)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - codec_id) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*codec_id); - - if(*codec_id != TNG_UNCOMPRESSED) - { - memcpy(multiplier, contents+*offset, - sizeof(*multiplier)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - (int64_t *) multiplier) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*multiplier); - } - else - { - *multiplier = 1; - } - - if(*dependency & TNG_FRAME_DEPENDENT) - { - if(*sparse_data) - { - memcpy(first_frame_with_data, contents+*offset, - sizeof(*first_frame_with_data)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - first_frame_with_data) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*first_frame_with_data); - - memcpy(stride_length, contents+*offset, - sizeof(*stride_length)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - stride_length) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*stride_length); - *n_frames = tng_data->current_trajectory_frame_set.n_frames - - (*first_frame_with_data - - tng_data->current_trajectory_frame_set.first_frame); - } - else - { - *first_frame_with_data = 0; - *stride_length = 1; - *n_frames = tng_data->current_trajectory_frame_set.n_frames; - } - } - else - { - *first_frame_with_data = 0; - *stride_length = 1; - *n_frames = 1; - } - - if (*dependency & TNG_PARTICLE_DEPENDENT) - { - memcpy(num_first_particle, contents+*offset, - sizeof(*num_first_particle)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - num_first_particle) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*num_first_particle); - - memcpy(block_n_particles, contents+*offset, - sizeof(*block_n_particles)); - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - block_n_particles) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } - *offset += sizeof(*block_n_particles); - } - - if(!block->block_contents) - { - free(contents); - } - return(TNG_SUCCESS); -} - -/** Read the contents of a data block (particle or non-particle data). - * @param tng_data is a trajectory data container. - * @param block is the block to store the data (should already contain - * the block headers). - * @param hash_mode is an option to decide whether to use the md5 hash or not. - * If hash_mode == TNG_USE_HASH the written md5 hash in the file will be - * compared to the md5 hash of the read contents to ensure valid data. - * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major - * error has occured. - */ -static tng_function_status tng_data_block_contents_read - (tng_trajectory_t tng_data, - tng_gen_block_t block, - const char hash_mode) -{ - int64_t n_values, codec_id, n_frames, first_frame_with_data; - int64_t stride_length, block_n_particles, num_first_particle; - double multiplier; - char datatype, dependency, sparse_data; - int offset = 0; - tng_bool same_hash; - - if(tng_input_file_init(tng_data) != TNG_SUCCESS) - { - return(TNG_CRITICAL); - } - - if(block->block_contents) - { - free(block->block_contents); - } - - block->block_contents = malloc(block->block_contents_size); - if(!block->block_contents) - { - fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", - block->block_contents_size, __FILE__, __LINE__); - return(TNG_CRITICAL); - } - - /* Read the whole block into block_contents to be able to write it to - * disk even if it cannot be interpreted. */ - if(fread(block->block_contents, block->block_contents_size, 1, - tng_data->input_file) == 0) - { - fprintf(stderr, "TNG library: Cannot read block. %s: %d\n", __FILE__, __LINE__); - return(TNG_CRITICAL); - } - - /* FIXME: Does not check if the size of the contents matches the expected - * size or if the contents can be read. */ - - if(hash_mode == TNG_USE_HASH) - { - tng_md5_hash_match_verify(block, &same_hash); - if(same_hash != TNG_TRUE) - { - fprintf(stderr, "TNG library: '%s' data block contents corrupt. Hashes do not match. %s: %d\n", - block->name, __FILE__, __LINE__); - /* return(TNG_FAILURE); */ - } - } - - if(tng_data_block_meta_information_read(tng_data, block, - &offset, &datatype, - &dependency, &sparse_data, - &n_values, &codec_id, - &first_frame_with_data, - &stride_length, &n_frames, - &num_first_particle, - &block_n_particles, - &multiplier) == TNG_CRITICAL) - { - fprintf(stderr, "TNG library: Cannot read data block (%s) meta information. %s: %d\n", - block->name, __FILE__, __LINE__); - return(TNG_CRITICAL); - } - - if (dependency & TNG_PARTICLE_DEPENDENT) - { - return(tng_particle_data_read(tng_data, block, - &offset, datatype, - num_first_particle, - block_n_particles, - first_frame_with_data, - stride_length, - n_frames, n_values, - codec_id, multiplier)); - } - else - { - return(tng_data_read(tng_data, block, - &offset, datatype, - first_frame_with_data, - stride_length, - n_frames, n_values, - codec_id, multiplier)); - } -} - -/** Update the md5 hash of a block already written to the file - * @param tng_data is a trajectory data container. - * @param block is the block, of which to update the md5 hash. - * @param header_start_pos is the file position where the block header starts. - * @param contents_start_pos is the file position where the block contents - * start. - * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major - * error has occured. - */ -static tng_function_status tng_md5_hash_update(tng_trajectory_t tng_data, - tng_gen_block_t block, - const int64_t header_start_pos, - const int64_t contents_start_pos) -{ - if(block->block_contents) - { - free(block->block_contents); + block->block_contents_size - data_start_pos); + if(stat != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Could not write gzipped block data. %s: %d\n", __FILE__, + __LINE__); + if(stat == TNG_CRITICAL) + { + return(TNG_CRITICAL); + } + data->codec_id = TNG_UNCOMPRESSED; + } + /* fprintf(stderr, "TNG library: After compression: %"PRId64"\n", block->block_contents_size); */ + break; +#endif + } } - block->block_contents = malloc(block->block_contents_size); - if(!block->block_contents) + if(tng_block_header_write(tng_data, block, hash_mode) != TNG_SUCCESS) { - fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", - block->block_contents_size, __FILE__, __LINE__); + fprintf(stderr, "TNG library: Cannot write header of file %s. %s: %d\n", + tng_data->output_file_path, __FILE__, __LINE__); return(TNG_CRITICAL); } - fseek(tng_data->output_file, (long)contents_start_pos, SEEK_SET); - if(fread(block->block_contents, block->block_contents_size, 1, - tng_data->output_file) == 0) + if(fwrite(block->block_contents, block->block_contents_size, 1, + tng_data->output_file) != 1) { - fprintf(stderr, "TNG library: Cannot read block. %s: %d\n", __FILE__, __LINE__); + fprintf(stderr, "TNG library: Could not write all block data. %s: %d\n", + __FILE__, __LINE__); return(TNG_CRITICAL); } - tng_block_md5_hash_generate(block); - - fseek(tng_data->output_file, (long)header_start_pos + 3 * sizeof(int64_t), - SEEK_SET); - fwrite(block->md5_hash, TNG_MD5_HASH_LEN, 1, tng_data->output_file); - return(TNG_SUCCESS); } -/** Update the frame set pointers in the file header (general info block), - * already written to disk +/** Read the meta information of a data block (particle or non-particle data). * @param tng_data is a trajectory data container. - * @param hash_mode specifies whether to update the block md5 hash when - * updating the pointers. + * @param block is the block to store the data (should already contain + * the block headers). * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major * error has occured. */ -static tng_function_status tng_header_pointers_update - (tng_trajectory_t tng_data, const char hash_mode) +static tng_function_status tng_data_block_meta_information_read + (tng_trajectory_t tng_data, + tng_gen_block_t block, + int *offset, + char *datatype, + char *dependency, + char *sparse_data, + int64_t *n_values, + int64_t *codec_id, + int64_t *first_frame_with_data, + int64_t *stride_length, + int64_t *n_frames, + int64_t *num_first_particle, + int64_t *block_n_particles, + double *multiplier) { - tng_gen_block_t block; - FILE *temp = tng_data->input_file; - int64_t output_file_pos, pos, contents_start_pos; + int meta_size; + char *contents; - if(tng_output_file_init(tng_data) != TNG_SUCCESS) + if(block->block_contents) { - fprintf(stderr, "TNG library: Cannot initialise destination file. %s: %d\n", - __FILE__, __LINE__); - return(TNG_CRITICAL); + contents = block->block_contents; } + else + { + meta_size = 3 * sizeof(char) + sizeof(double) + 6 * sizeof(int64_t); + contents = malloc(meta_size); + if(!contents) + { + fprintf(stderr, "TNG library: Cannot allocate memory (%d bytes). %s: %d\n", + meta_size, __FILE__, __LINE__); + } - tng_data->input_file = tng_data->output_file; + if(fread(contents, meta_size, 1, tng_data->input_file) == 0) + { + fprintf(stderr, "TNG library: Cannot read data block meta information. %s: %d\n", __FILE__, __LINE__); + free(contents); + return(TNG_CRITICAL); + } + } - tng_block_init(&block); + memcpy(datatype, contents+*offset, + sizeof(*datatype)); + *offset += sizeof(*datatype); - output_file_pos = ftell(tng_data->output_file); - fseek(tng_data->output_file, 0, SEEK_SET); + memcpy(dependency, contents+*offset, + sizeof(*dependency)); + *offset += sizeof(*dependency); - if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) + if(*dependency & TNG_FRAME_DEPENDENT) { - fprintf(stderr, "TNG library: Cannot read general info header. %s: %d\n", - __FILE__, __LINE__); - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); + memcpy(sparse_data, contents+*offset, + sizeof(*sparse_data)); + *offset += sizeof(*sparse_data); } - contents_start_pos = ftell(tng_data->output_file); - - fseek(tng_data->output_file, (long)block->block_contents_size - 5 * - sizeof(int64_t), SEEK_CUR); - - tng_data->input_file = temp; - - pos = tng_data->first_trajectory_frame_set_output_file_pos; - + memcpy(n_values, contents+*offset, + sizeof(*n_values)); if(tng_data->input_endianness_swap_func_64) { if(tng_data->input_endianness_swap_func_64(tng_data, - &pos) + n_values) != TNG_SUCCESS) { fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", __FILE__, __LINE__); } } + *offset += sizeof(*n_values); - if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) - { - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - - pos = tng_data->last_trajectory_frame_set_output_file_pos; - + memcpy(codec_id, contents+*offset, + sizeof(*codec_id)); if(tng_data->input_endianness_swap_func_64) { if(tng_data->input_endianness_swap_func_64(tng_data, - &pos) + codec_id) != TNG_SUCCESS) { fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", __FILE__, __LINE__); } } + *offset += sizeof(*codec_id); - if(fwrite(&pos, - sizeof(int64_t), 1, tng_data->output_file) != 1) - { - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - - if(hash_mode == TNG_USE_HASH) + if(*codec_id != TNG_UNCOMPRESSED) { - tng_md5_hash_update(tng_data, block, 0, contents_start_pos); + memcpy(multiplier, contents+*offset, + sizeof(*multiplier)); + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + (int64_t *) multiplier) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } + } + *offset += sizeof(*multiplier); } - - tng_block_destroy(&block); - - fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); - - return(TNG_SUCCESS); -} - -/** Update the frame set pointers in the current frame set block, already - * written to disk. It also updates the pointers of the blocks pointing to - * the current frame set block. - * @param tng_data is a trajectory data container. - * @param hash_mode specifies whether to update the block md5 hash when - * updating the pointers. - * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major - * error has occured. - */ -static tng_function_status tng_frame_set_pointers_update - (tng_trajectory_t tng_data, const char hash_mode) -{ - tng_gen_block_t block; - tng_trajectory_frame_set_t frame_set; - FILE *temp = tng_data->input_file; - int64_t pos, output_file_pos, header_start_pos, contents_start_pos; - - if(tng_output_file_init(tng_data) != TNG_SUCCESS) + else { - fprintf(stderr, "TNG library: Cannot initialise destination file. %s: %d\n", - __FILE__, __LINE__); - return(TNG_CRITICAL); + *multiplier = 1; } - tng_block_init(&block); - output_file_pos = ftell(tng_data->output_file); - - tng_data->input_file = tng_data->output_file; - - frame_set = &tng_data->current_trajectory_frame_set; - - /* Update previous frame set */ - if(frame_set->prev_frame_set_file_pos != -1 && - frame_set->prev_frame_set_file_pos != 0) + if(*dependency & TNG_FRAME_DEPENDENT) { - fseek(tng_data->output_file, (long)frame_set->prev_frame_set_file_pos, - SEEK_SET); - - header_start_pos = frame_set->prev_frame_set_file_pos; + if(*sparse_data) + { + memcpy(first_frame_with_data, contents+*offset, + sizeof(*first_frame_with_data)); + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + first_frame_with_data) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } + } + *offset += sizeof(*first_frame_with_data); - if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) + memcpy(stride_length, contents+*offset, + sizeof(*stride_length)); + if(tng_data->input_endianness_swap_func_64) + { + if(tng_data->input_endianness_swap_func_64(tng_data, + stride_length) + != TNG_SUCCESS) + { + fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", + __FILE__, __LINE__); + } + } + *offset += sizeof(*stride_length); + *n_frames = tng_data->current_trajectory_frame_set.n_frames - + (*first_frame_with_data - + tng_data->current_trajectory_frame_set.first_frame); + } + else { - fprintf(stderr, "TNG library: Cannot read frame header. %s: %d\n", - __FILE__, __LINE__); - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); + *first_frame_with_data = 0; + *stride_length = 1; + *n_frames = tng_data->current_trajectory_frame_set.n_frames; } + } + else + { + *first_frame_with_data = 0; + *stride_length = 1; + *n_frames = 1; + } - contents_start_pos = ftell(tng_data->output_file); - - fseek(tng_data->output_file, (long)block->block_contents_size - (6 * - sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); - - pos = tng_data->current_trajectory_frame_set_output_file_pos; - + if (*dependency & TNG_PARTICLE_DEPENDENT) + { + memcpy(num_first_particle, contents+*offset, + sizeof(*num_first_particle)); if(tng_data->input_endianness_swap_func_64) { if(tng_data->input_endianness_swap_func_64(tng_data, - &pos) + num_first_particle) != TNG_SUCCESS) { fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", __FILE__, __LINE__); } } + *offset += sizeof(*num_first_particle); - if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) - { - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - - if(hash_mode == TNG_USE_HASH) - { - tng_md5_hash_update(tng_data, block, header_start_pos, - contents_start_pos); - } - fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); - } - - /* Update the frame set one medium stride step before */ - if(frame_set->medium_stride_prev_frame_set_file_pos != -1 && - frame_set->medium_stride_prev_frame_set_file_pos != 0) - { - fseek(tng_data->output_file, - (long)frame_set->medium_stride_prev_frame_set_file_pos, - SEEK_SET); - - if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot read frame set header. %s: %d\n", - __FILE__, __LINE__); - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - - contents_start_pos = ftell(tng_data->output_file); - - fseek(tng_data->output_file, (long)block->block_contents_size - (4 * - sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); - - pos = tng_data->current_trajectory_frame_set_output_file_pos; - + memcpy(block_n_particles, contents+*offset, + sizeof(*block_n_particles)); if(tng_data->input_endianness_swap_func_64) { if(tng_data->input_endianness_swap_func_64(tng_data, - &pos) + block_n_particles) != TNG_SUCCESS) { fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", __FILE__, __LINE__); } } - - if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) - { - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); - } - - if(hash_mode == TNG_USE_HASH) - { - tng_md5_hash_update(tng_data, block, - frame_set->medium_stride_prev_frame_set_file_pos, - contents_start_pos); - } + *offset += sizeof(*block_n_particles); } - /* Update the frame set one long stride step before */ - if(frame_set->long_stride_prev_frame_set_file_pos != -1 && - frame_set->long_stride_prev_frame_set_file_pos != 0) + if(!block->block_contents) { - fseek(tng_data->output_file, - (long)frame_set->long_stride_prev_frame_set_file_pos, - SEEK_SET); + free(contents); + } + return(TNG_SUCCESS); +} - if(tng_block_header_read(tng_data, block) != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot read frame set header. %s: %d\n", - __FILE__, __LINE__); - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); - } +/** Read the contents of a data block (particle or non-particle data). + * @param tng_data is a trajectory data container. + * @param block is the block to store the data (should already contain + * the block headers). + * @param hash_mode is an option to decide whether to use the md5 hash or not. + * If hash_mode == TNG_USE_HASH the written md5 hash in the file will be + * compared to the md5 hash of the read contents to ensure valid data. + * @return TNG_SUCCESS (0) if successful or TNG_CRITICAL (2) if a major + * error has occured. + */ +static tng_function_status tng_data_block_contents_read + (tng_trajectory_t tng_data, + tng_gen_block_t block, + const char hash_mode) +{ + int64_t n_values, codec_id, n_frames, first_frame_with_data; + int64_t stride_length, block_n_particles, num_first_particle; + double multiplier; + char datatype, dependency, sparse_data; + int offset = 0; + tng_bool same_hash; - contents_start_pos = ftell(tng_data->output_file); + if(tng_input_file_init(tng_data) != TNG_SUCCESS) + { + return(TNG_CRITICAL); + } - fseek(tng_data->output_file, (long)block->block_contents_size - (2 * - sizeof(int64_t) + 2 * sizeof(double)), SEEK_CUR); + if(block->block_contents) + { + free(block->block_contents); + } - pos = tng_data->current_trajectory_frame_set_output_file_pos; + block->block_contents = malloc(block->block_contents_size); + if(!block->block_contents) + { + fprintf(stderr, "TNG library: Cannot allocate memory (%"PRId64" bytes). %s: %d\n", + block->block_contents_size, __FILE__, __LINE__); + return(TNG_CRITICAL); + } - if(tng_data->input_endianness_swap_func_64) - { - if(tng_data->input_endianness_swap_func_64(tng_data, - &pos) - != TNG_SUCCESS) - { - fprintf(stderr, "TNG library: Cannot swap byte order. %s: %d\n", - __FILE__, __LINE__); - } - } + /* Read the whole block into block_contents to be able to write it to + * disk even if it cannot be interpreted. */ + if(fread(block->block_contents, block->block_contents_size, 1, + tng_data->input_file) == 0) + { + fprintf(stderr, "TNG library: Cannot read block. %s: %d\n", __FILE__, __LINE__); + return(TNG_CRITICAL); + } - if(fwrite(&pos, sizeof(int64_t), 1, tng_data->output_file) != 1) - { - tng_data->input_file = temp; - tng_block_destroy(&block); - return(TNG_CRITICAL); - } + /* FIXME: Does not check if the size of the contents matches the expected + * size or if the contents can be read. */ - if(hash_mode == TNG_USE_HASH) + if(hash_mode == TNG_USE_HASH) + { + tng_md5_hash_match_verify(block, &same_hash); + if(same_hash != TNG_TRUE) { - tng_md5_hash_update(tng_data, block, - frame_set->long_stride_prev_frame_set_file_pos, - contents_start_pos); + fprintf(stderr, "TNG library: '%s' data block contents corrupt. Hashes do not match. %s: %d\n", + block->name, __FILE__, __LINE__); + /* return(TNG_FAILURE); */ } } - fseek(tng_data->output_file, (long)output_file_pos, SEEK_SET); - - tng_data->input_file = temp; - - tng_block_destroy(&block); + if(tng_data_block_meta_information_read(tng_data, block, + &offset, &datatype, + &dependency, &sparse_data, + &n_values, &codec_id, + &first_frame_with_data, + &stride_length, &n_frames, + &num_first_particle, + &block_n_particles, + &multiplier) == TNG_CRITICAL) + { + fprintf(stderr, "TNG library: Cannot read data block (%s) meta information. %s: %d\n", + block->name, __FILE__, __LINE__); + return(TNG_CRITICAL); + } - return(TNG_SUCCESS); + if (dependency & TNG_PARTICLE_DEPENDENT) + { + return(tng_particle_data_read(tng_data, block, + &offset, datatype, + num_first_particle, + block_n_particles, + first_frame_with_data, + stride_length, + n_frames, n_values, + codec_id, multiplier)); + } + else + { + return(tng_data_read(tng_data, block, + &offset, datatype, + first_frame_with_data, + stride_length, + n_frames, n_values, + codec_id, multiplier)); + } } + /* // ** Move the blocks in a frame set so that there is no unused space between // * them. This can only be done on the last frame set in the file and should @@ -9699,6 +9731,11 @@ tng_function_status DECLSPECDLLEXPORT tng_trajectory_destroy(tng_trajectory_t *t if(tng_data->input_file) { + if(tng_data->output_file == tng_data->input_file) + { + tng_frame_set_finalize(tng_data, TNG_USE_HASH); + tng_data->output_file = 0; + } fclose(tng_data->input_file); tng_data->input_file = 0; } @@ -10291,6 +10328,7 @@ tng_function_status DECLSPECDLLEXPORT tng_output_append_file_set tng_data->output_file_path, __FILE__, __LINE__); return(TNG_CRITICAL); } + tng_data->input_file = tng_data->output_file; return(TNG_SUCCESS); } @@ -12307,11 +12345,55 @@ static TNG_INLINE tng_function_status tng_particle_mapping_get_real_particle } */ +static tng_function_status DECLSPECDLLEXPORT tng_file_headers_len_get + (tng_trajectory_t tng_data, + int64_t *len) +{ + int64_t orig_pos; + tng_gen_block_t block; + + TNG_ASSERT(tng_data, "TNG library: Trajectory container not properly setup."); + + if(tng_input_file_init(tng_data) != TNG_SUCCESS) + { + return(TNG_CRITICAL); + } + + *len = 0; + + orig_pos = ftell(tng_data->input_file); + + if(!tng_data->input_file_len) + { + fseek(tng_data->input_file, 0, SEEK_END); + tng_data->input_file_len = ftell(tng_data->input_file); + } + fseek(tng_data->input_file, 0, SEEK_SET); + + tng_block_init(&block); + /* Read through the headers of non-trajectory blocks (they come before the + * trajectory blocks in the file) */ + while (*len < tng_data->input_file_len && + tng_block_header_read(tng_data, block) != TNG_CRITICAL && + block->id != -1 && + block->id != TNG_TRAJECTORY_FRAME_SET) + { + *len += block->header_contents_size + block->block_contents_size; + fseek(tng_data->input_file, block->block_contents_size, SEEK_CUR); + } + + fseek(tng_data->input_file, orig_pos, SEEK_SET); + + tng_block_destroy(&block); + + return(TNG_SUCCESS); +} + tng_function_status DECLSPECDLLEXPORT tng_file_headers_read (tng_trajectory_t tng_data, const char hash_mode) { - int cnt = 0, prev_pos = 0; + int64_t prev_pos = 0; tng_gen_block_t block; TNG_ASSERT(tng_data, "TNG library: Trajectory container not properly setup."); @@ -12327,8 +12409,8 @@ tng_function_status DECLSPECDLLEXPORT tng_file_headers_read { fseek(tng_data->input_file, 0, SEEK_END); tng_data->input_file_len = ftell(tng_data->input_file); - fseek(tng_data->input_file, 0, SEEK_SET); } + fseek(tng_data->input_file, 0, SEEK_SET); tng_block_init(&block); /* Non trajectory blocks (they come before the trajectory @@ -12338,11 +12420,7 @@ tng_function_status DECLSPECDLLEXPORT tng_file_headers_read block->id != -1 && block->id != TNG_TRAJECTORY_FRAME_SET) { - if(tng_block_read_next(tng_data, block, - hash_mode) == TNG_SUCCESS) - { - cnt++; - } + tng_block_read_next(tng_data, block, hash_mode); prev_pos = ftell(tng_data->input_file); } @@ -12362,7 +12440,7 @@ tng_function_status DECLSPECDLLEXPORT tng_file_headers_write const char hash_mode) { int i; - int64_t len, tot_len = 0, data_start_pos; + int64_t len, orig_len, tot_len = 0, data_start_pos; tng_gen_block_t block; TNG_ASSERT(tng_data, "TNG library: Trajectory container not properly setup."); @@ -12374,6 +12452,8 @@ tng_function_status DECLSPECDLLEXPORT tng_file_headers_write if(tng_data->n_trajectory_frame_sets > 0) { + tng_file_headers_len_get(tng_data, &orig_len); + tng_block_init(&block); block->name = malloc(TNG_MAX_STR_LEN); if(!block->name) @@ -12419,8 +12499,14 @@ tng_function_status DECLSPECDLLEXPORT tng_file_headers_write &len); tot_len += len; } - tng_block_destroy(&block); + + if(tot_len > orig_len) + { + tng_migrate_data_in_file(tng_data, orig_len+1, tot_len - orig_len); + } + + tng_data->current_trajectory_frame_set_output_file_pos = -1; } /* TODO: If there is already frame set data written to this file (e.g. when @@ -16652,9 +16738,6 @@ tng_function_status DECLSPECDLLEXPORT tng_util_trajectory_open (*tng_data_p)->last_trajectory_frame_set_input_file_pos; (*tng_data_p)->current_trajectory_frame_set_output_file_pos = (*tng_data_p)->current_trajectory_frame_set_input_file_pos; - (*tng_data_p)->first_trajectory_frame_set_input_file_pos = -1; - (*tng_data_p)->last_trajectory_frame_set_input_file_pos = -1; - (*tng_data_p)->current_trajectory_frame_set_input_file_pos = -1; if((*tng_data_p)->input_file) { fclose((*tng_data_p)->input_file); diff --git a/src/tests/tng_io_testing.c b/src/tests/tng_io_testing.c index 17718d0..dd13d92 100644 --- a/src/tests/tng_io_testing.c +++ b/src/tests/tng_io_testing.c @@ -1,7 +1,7 @@ /* This code is part of the tng binary trajectory format. * * Written by Magnus Lundborg - * Copyright (c) 2012-2013, The GROMACS development team. + * Copyright (c) 2012-2014, The GROMACS development team. * Check out http://www.gromacs.org for more information. * * @@ -147,6 +147,9 @@ static tng_function_status tng_test_write_and_read_traj(tng_trajectory_t *traj) tng_medium_stride_length_set(*traj, 10); tng_long_stride_length_set(*traj, 100); + tng_first_user_name_set(*traj, "User1"); + tng_first_program_name_set(*traj, "tng_testing"); + /* Create molecules */ if(tng_test_setup_molecules(*traj) == TNG_CRITICAL) { @@ -210,7 +213,7 @@ static tng_function_status tng_test_write_and_read_traj(tng_trajectory_t *traj) /* Generate a custom annotation data block */ strcpy(annotation, "This trajectory was generated from tng_io_testing. " "It is not a real MD trajectory."); - if(tng_data_block_add(*traj, 10100, "DETAILS", TNG_CHAR_DATA, + if(tng_data_block_add(*traj, TNG_TRAJ_GENERAL_COMMENTS, "COMMENTS", TNG_CHAR_DATA, TNG_NON_TRAJECTORY_BLOCK, 1, 1, 1, TNG_UNCOMPRESSED, annotation) != TNG_SUCCESS) { @@ -567,6 +570,26 @@ tng_function_status tng_test_get_positions_data(tng_trajectory_t traj) return(TNG_SUCCESS); } + +tng_function_status tng_test_append(tng_trajectory_t traj) +{ + tng_function_status stat; + + stat = tng_util_trajectory_open(TNG_EXAMPLE_FILES_DIR "tng_test.tng", 'a', &traj); + if(stat != TNG_SUCCESS) + { + return(stat); + } + + tng_last_user_name_set(traj, "User2"); + tng_last_program_name_set(traj, "tng_testing"); + tng_file_headers_write(traj, TNG_USE_HASH); + + stat = tng_util_trajectory_close(&traj); + + return(stat); +} + int main() { tng_trajectory_t traj; @@ -681,6 +704,17 @@ int main() printf("Test Utility function close:\t\t\tSucceeded.\n"); } + if(tng_test_append(traj) != TNG_SUCCESS) + { + printf("Test Append:\t\t\t\t\tFailed. %s: %d.\n", + __FILE__, __LINE__); + exit(1); + } + else + { + printf("Test Append:\t\t\t\t\tSucceeded.\n"); + } + printf("Tests finished\n"); exit(0); -- cgit v0.10.1