From 1b6d0a2638d0d74a3dfba12735d5281208f77f99 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Tue, 25 Feb 2025 17:49:31 +0900 Subject: [PATCH] Impl. file format v 4 write This is actually part of an original commit that was broken into two pieces due to weird git shennanigans (newly added code was not properly "diff"ed by git). --- src/archiver.c | 1172 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1172 insertions(+) diff --git a/src/archiver.c b/src/archiver.c index a4cac06..ba5147e 100644 --- a/src/archiver.c +++ b/src/archiver.c @@ -2251,6 +2251,8 @@ int simple_archiver_write_all(FILE *out_f, SDArchiverState *state, return simple_archiver_write_v2(out_f, state, filenames); case 3: return simple_archiver_write_v3(out_f, state, filenames); + case 4: + return simple_archiver_write_v4(out_f, state, filenames); default: fprintf(stderr, "ERROR: Unsupported write version %" PRIu32 "!\n", state->parsed->write_version); @@ -5653,6 +5655,1176 @@ int simple_archiver_write_v3(FILE *out_f, SDArchiverState *state, return SDAS_SUCCESS; } +int simple_archiver_write_v4(FILE *out_f, SDArchiverState *state, + const SDArchiverLinkedList *filenames) { + fprintf(stderr, "Writing archive of file format 4\n"); + + // First create a "set" of absolute paths to given filenames. + __attribute__((cleanup(simple_archiver_hash_map_free))) + SDArchiverHashMap *abs_filenames = simple_archiver_hash_map_init(); + void **ptr_array = malloc(sizeof(void *) * 2); + ptr_array[0] = abs_filenames; + ptr_array[1] = (void *)state->parsed->user_cwd; + if (simple_archiver_list_get(filenames, filenames_to_abs_map_fn, ptr_array)) { + free(ptr_array); + return SDAS_FAILED_TO_CREATE_MAP; + } + free(ptr_array); + + // Get a list of symlinks and a list of files. + __attribute__((cleanup(simple_archiver_list_free))) + SDArchiverLinkedList *symlinks_list = simple_archiver_list_init(); + __attribute__((cleanup(simple_archiver_list_free))) + SDArchiverLinkedList *files_list = simple_archiver_list_init(); + __attribute__((cleanup(simple_archiver_list_free))) + SDArchiverLinkedList *dirs_list = simple_archiver_list_init(); + __attribute__((cleanup(simple_archiver_priority_heap_free))) + SDArchiverPHeap *files_pheap = + (state->parsed->flags & 0x40) + ? simple_archiver_priority_heap_init_less_fn(greater_fn) + : NULL; + uint64_t from_files_count = 0; + + ptr_array = malloc(sizeof(void *) * 7); + ptr_array[0] = symlinks_list; + ptr_array[1] = files_list; + ptr_array[2] = (void *)state->parsed->user_cwd; + ptr_array[3] = files_pheap; + ptr_array[4] = dirs_list; + ptr_array[5] = state; + ptr_array[6] = &from_files_count; + + if (simple_archiver_list_get(filenames, symlinks_and_files_from_files, + ptr_array)) { + free(ptr_array); + return SDAS_INTERNAL_ERROR; + } + free(ptr_array); + + if (files_pheap) { + while (files_pheap->size > 0) { + simple_archiver_list_add(files_list, + simple_archiver_priority_heap_pop(files_pheap), + free_internal_file_info); + } + simple_archiver_priority_heap_free(&files_pheap); + } + + if (symlinks_list->count + + files_list->count + + dirs_list->count != from_files_count) { + fprintf(stderr, + "ERROR: Count mismatch between files and symlinks and files from " + "parser!\n"); + return SDAS_INTERNAL_ERROR; + } + + if (fwrite("SIMPLE_ARCHIVE_VER", 1, 18, out_f) != 18) { + return SDAS_FAILED_TO_WRITE; + } + + char buf[SIMPLE_ARCHIVER_BUFFER_SIZE]; + uint16_t u16 = 4; + + simple_archiver_helper_16_bit_be(&u16); + + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + const size_t prefix_length = state->parsed->prefix + ? strlen(state->parsed->prefix) + : 0; + + if (state->parsed->compressor && !state->parsed->decompressor) { + return SDAS_NO_DECOMPRESSOR; + } else if (!state->parsed->compressor && state->parsed->decompressor) { + return SDAS_NO_COMPRESSOR; + } else if (state->parsed->compressor && state->parsed->decompressor) { + // 4 bytes flags, using de/compressor. + memset(buf, 0, 4); + buf[0] |= 1; + if (fwrite(buf, 1, 4, out_f) != 4) { + return SDAS_FAILED_TO_WRITE; + } + + size_t len = strlen(state->parsed->compressor); + if (len >= 0xFFFF) { + fprintf(stderr, "ERROR: Compressor cmd is too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)len; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 1, 2, out_f) != 2) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + + if (fwrite(state->parsed->compressor, 1, u16 + 1, out_f) != + (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + + len = strlen(state->parsed->decompressor); + if (len >= 0xFFFF) { + fprintf(stderr, "ERROR: Decompressor cmd is too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)len; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 1, 2, out_f) != 2) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + + if (fwrite(state->parsed->decompressor, 1, u16 + 1, out_f) != + (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + // 4 bytes flags, not using de/compressor. + memset(buf, 0, 4); + if (fwrite(buf, 1, 4, out_f) != 4) { + return SDAS_FAILED_TO_WRITE; + } + } + + uint32_t u32; + uint64_t u64 = symlinks_list->count; + simple_archiver_helper_64_bit_be(&u64); + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_64_bit_be(&u64); + + // Change cwd if user specified. + __attribute__((cleanup( + simple_archiver_helper_cleanup_chdir_back))) char *original_cwd = NULL; + if (state->parsed->user_cwd) { + original_cwd = realpath(".", NULL); + if (chdir(state->parsed->user_cwd)) { + return SDAS_INTERNAL_ERROR; + } + } + + { + const SDArchiverLLNode *node = symlinks_list->head; + uint64_t idx; + for (idx = 0; + idx < symlinks_list->count && node != symlinks_list->tail;) { + node = node->next; + ++idx; + memset(buf, 0, 2); + + uint_fast8_t is_invalid = 0; + + __attribute__((cleanup( + simple_archiver_helper_cleanup_malloced))) void *abs_path = NULL; + __attribute__((cleanup( + simple_archiver_helper_cleanup_malloced))) void *rel_path = NULL; + if ((state->parsed->flags & 0x100) != 0) { + // Preserve symlink target. + char *path_buf = malloc(1024); + ssize_t ret = readlink(node->data, path_buf, 1023); + if (ret == -1) { + fprintf(stderr, "WARNING: Failed to get symlink's target!\n"); + free(path_buf); + is_invalid = 1; + } else { + path_buf[ret] = 0; + if (path_buf[0] == '/') { + abs_path = path_buf; + buf[0] |= 1; + } else { + rel_path = path_buf; + } + } + } else { + abs_path = realpath(node->data, NULL); + // Check if symlink points to thing to be stored into archive. + if (abs_path) { + __attribute__((cleanup( + simple_archiver_helper_cleanup_malloced))) void *link_abs_path = + simple_archiver_helper_real_path_to_name(node->data); + if (!link_abs_path) { + fprintf(stderr, "WARNING: Failed to get absolute path to link!\n"); + } else { + rel_path = simple_archiver_filenames_to_relative_path(link_abs_path, + abs_path); + } + } + } + + if (abs_path && (state->parsed->flags & 0x20) == 0 && + (state->parsed->flags & 0x100) == 0 && + !simple_archiver_hash_map_get(abs_filenames, abs_path, + strlen(abs_path) + 1)) { + // Is not a filename being archived. + buf[1] |= 8; + if ((state->parsed->flags & 0x80) == 0) { + // Not a "safe link", mark invalid and continue. + is_invalid = 1; + fprintf(stderr, + "WARNING: \"%s\" points to outside of archived files (or is " + "invalid) and \"--no-safe-links\" not specified, will not " + "store abs/rel-links to this entry!\n", + (const char *)node->data); + } else { + // Safe links disabled, set preference to absolute path. + buf[0] |= 1; + } + } else if ((state->parsed->flags & 0x100) != 0 && + (state->parsed->flags & 0x80) == 0 && !is_invalid) { + __attribute__((cleanup( + simple_archiver_helper_cleanup_c_string))) char *target_realpath = + realpath(node->data, NULL); + if (!target_realpath) { + fprintf( + stderr, + "WARNING: \"%s\" is an invalid symlink and \"--no-safe-links\" " + "not specified, will skip this symlink!\n", + (const char *)node->data); + is_invalid = 1; + } else if (!simple_archiver_hash_map_get(abs_filenames, target_realpath, + strlen(target_realpath) + 1)) { + fprintf( + stderr, + "WARNING: \"%s\" points to outside of archived files and " + "\"--no-safe-links\" not specified, will skip this symlink!\n", + (const char *)node->data); + is_invalid = 1; + } + } + + if (!abs_path && !rel_path) { + // No valid paths, mark as invalid. + fprintf(stderr, + "WARNING: \"%s\" is an invalid symlink, will not store rel/abs " + "link paths!\n", + (const char *)node->data); + is_invalid = 1; + } + + // Get symlink stats for permissions. + struct stat stat_buf; + memset(&stat_buf, 0, sizeof(struct stat)); + int stat_status = + fstatat(AT_FDCWD, node->data, &stat_buf, AT_SYMLINK_NOFOLLOW); + if (stat_status != 0) { + return SDAS_INTERNAL_ERROR; + } + + if ((stat_buf.st_mode & S_IRUSR) != 0) { + buf[0] |= 2; + } + if ((stat_buf.st_mode & S_IWUSR) != 0) { + buf[0] |= 4; + } + if ((stat_buf.st_mode & S_IXUSR) != 0) { + buf[0] |= 8; + } + if ((stat_buf.st_mode & S_IRGRP) != 0) { + buf[0] |= 0x10; + } + if ((stat_buf.st_mode & S_IWGRP) != 0) { + buf[0] |= 0x20; + } + if ((stat_buf.st_mode & S_IXGRP) != 0) { + buf[0] |= 0x40; + } + if ((stat_buf.st_mode & S_IROTH) != 0) { + buf[0] |= (char)0x80; + } + if ((stat_buf.st_mode & S_IWOTH) != 0) { + buf[1] |= 1; + } + if ((stat_buf.st_mode & S_IXOTH) != 0) { + buf[1] |= 2; + } + + if (is_invalid) { + buf[1] |= 4; + } + + if (fwrite(buf, 1, 2, out_f) != 2) { + return SDAS_FAILED_TO_WRITE; + } + + const size_t link_length = strlen(node->data); + size_t len = link_length; + if (state->parsed->prefix) { + len += prefix_length; + } + if (len >= 0xFFFF) { + fprintf(stderr, "ERROR: Link name is too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)len; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (state->parsed->prefix) { + size_t fwrite_ret = fwrite(state->parsed->prefix, + 1, + prefix_length, + out_f); + fwrite_ret += fwrite(node->data, 1, link_length + 1, out_f); + if (fwrite_ret != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else if (fwrite(node->data, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + + if (abs_path && (state->parsed->flags & 0x20) == 0 && !is_invalid) { + if (state->parsed->prefix) { + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *abs_path_prefixed = + simple_archiver_helper_insert_prefix_in_link_path( + state->parsed->prefix, node->data, abs_path); + if (!abs_path_prefixed) { + fprintf(stderr, + "ERROR: Failed to add prefix to abs symlink!\n"); + return SDAS_INTERNAL_ERROR; + } + const size_t abs_path_pref_length = strlen(abs_path_prefixed); + if (abs_path_pref_length >= 0xFFFF) { + fprintf(stderr, + "ERROR: Symlink destination absolute path with prefix is " + "too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)abs_path_pref_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(abs_path_prefixed, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + const size_t abs_path_length = strlen(abs_path); + if (abs_path_length >= 0xFFFF) { + fprintf(stderr, + "ERROR: Symlink destination absolute path is too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)abs_path_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(abs_path, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + if (rel_path && !is_invalid) { + if (state->parsed->prefix) { + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *rel_path_prefixed = + simple_archiver_helper_insert_prefix_in_link_path( + state->parsed->prefix, node->data, rel_path); + if (!rel_path_prefixed) { + fprintf(stderr, + "ERROR: Failed to add prefix to relative symlink!\n"); + return SDAS_INTERNAL_ERROR; + } + const size_t rel_path_pref_length = strlen(rel_path_prefixed); + if (rel_path_pref_length >= 0xFFFF) { + fprintf(stderr, + "ERROR: Symlink destination relative path with prefix is " + "too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)rel_path_pref_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(rel_path_prefixed, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + len = strlen(rel_path); + if (len >= 0xFFFF) { + fprintf(stderr, + "ERROR: Symlink destination relative path is too long!\n"); + return SDAS_INVALID_PARSED_STATE; + } + + u16 = (uint16_t)len; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(rel_path, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + u32 = stat_buf.st_uid; + if (state->parsed->flags & 0x400) { + u32 = state->parsed->uid; + } else { + uint32_t mapped_uid; + if (simple_archiver_get_uid_mapping(state->parsed->mappings, + state->parsed->users_infos, + u32, + &mapped_uid, + NULL) == 0) { + u32 = mapped_uid; + } + } + simple_archiver_helper_32_bit_be(&u32); + if (fwrite(&u32, 4, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + u32 = stat_buf.st_gid; + if (state->parsed->flags & 0x800) { + u32 = state->parsed->gid; + } else { + uint32_t mapped_gid; + if (simple_archiver_get_gid_mapping(state->parsed->mappings, + state->parsed->users_infos, + u32, + &mapped_gid, + NULL) == 0) { + u32 = mapped_gid; + } + } + simple_archiver_helper_32_bit_be(&u32); + if (fwrite(&u32, 4, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + u32 = stat_buf.st_uid; + if (state->parsed->flags & 0x400) { + u32 = state->parsed->uid; + } + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *to_cleanup_user = NULL; + const char *username = simple_archiver_hash_map_get( + state->parsed->users_infos.UidToUname, + &u32, + sizeof(uint32_t)); + if (username) { + if ((state->parsed->flags & 0x400) == 0) { + uint32_t out_uid; + const char *mapped_user = NULL; + if (simple_archiver_get_user_mapping(state->parsed->mappings, + state->parsed->users_infos, + username, + &out_uid, + &mapped_user) == 0 + && mapped_user) { + username = mapped_user; + to_cleanup_user = (char *)mapped_user; + } + } + unsigned long name_length = strlen(username); + if (name_length > 0xFFFF) { + return SDAS_INTERNAL_ERROR; + } + u16 = (uint16_t)name_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(username, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + u32 = stat_buf.st_gid; + if (state->parsed->flags & 0x800) { + u32 = state->parsed->gid; + } + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *to_cleanup_group = NULL; + const char *groupname = simple_archiver_hash_map_get( + state->parsed->users_infos.GidToGname, + &u32, + sizeof(uint32_t)); + if (groupname) { + if ((state->parsed->flags & 0x800) == 0) { + uint32_t out_gid; + const char *mapped_group = NULL; + if (simple_archiver_get_group_mapping(state->parsed->mappings, + state->parsed->users_infos, + groupname, + &out_gid, + &mapped_group) == 0 + && mapped_group) { + groupname = mapped_group; + to_cleanup_group = (char *)mapped_group; + } + } + unsigned long group_length = strlen(groupname); + if (group_length > 0xFFFF) { + return SDAS_INTERNAL_ERROR; + } + u16 = (uint16_t)group_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(groupname, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + } + if (idx != symlinks_list->count) { + fprintf(stderr, + "ERROR: Iterated through %" PRIu64 " symlinks out of %" PRIu64 + " total!\n", + idx, symlinks_list->count); + return SDAS_INTERNAL_ERROR; + } + } + + if (is_sig_int_occurred) { + return SDAS_SIGINT; + } + + __attribute__((cleanup(simple_archiver_list_free))) + SDArchiverLinkedList *chunk_counts = simple_archiver_list_init(); + + { + uint64_t current_size = 0; + uint64_t current_count = 0; + void **ptrs = malloc(sizeof(void *) * 4); + ptrs[0] = (void *)&state->parsed->minimum_chunk_size; + ptrs[1] = ¤t_size; + ptrs[2] = ¤t_count; + ptrs[3] = chunk_counts; + if (simple_archiver_list_get(files_list, files_to_chunk_count, ptrs)) { + free(ptrs); + fprintf(stderr, "ERROR: Internal error calculating chunk counts!\n"); + return SDAS_INTERNAL_ERROR; + } + free(ptrs); + if ((chunk_counts->count == 0 || current_size > 0) && current_count > 0) { + uint64_t *count = malloc(sizeof(uint64_t)); + *count = current_count; + simple_archiver_list_add(chunk_counts, count, NULL); + } + } + + // Verify chunk counts. + { + uint64_t count = 0; + for (SDArchiverLLNode *node = chunk_counts->head->next; + node != chunk_counts->tail; node = node->next) { + if (*((uint64_t *)node->data) > 0xFFFFFFFF) { + fprintf(stderr, "ERROR: file count in chunk is too large!\n"); + return SDAS_INTERNAL_ERROR; + } + count += *((uint64_t *)node->data); + } + if (count != files_list->count) { + fprintf(stderr, + "ERROR: Internal error calculating chunk counts (invalid number " + "of files)!\n"); + return SDAS_INTERNAL_ERROR; + } + } + + // Write number of chunks. + u64 = chunk_counts->count; + simple_archiver_helper_64_bit_be(&u64); + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + __attribute__((cleanup(simple_archiver_helper_cleanup_malloced))) void + *non_compressing_chunk_size = NULL; + if (!state->parsed->compressor || !state->parsed->decompressor) { + non_compressing_chunk_size = malloc(sizeof(uint64_t)); + } + uint64_t *non_c_chunk_size = non_compressing_chunk_size; + + SDArchiverLLNode *file_node = files_list->head; + uint64_t chunk_count = 0; + for (SDArchiverLLNode *chunk_c_node = chunk_counts->head->next; + chunk_c_node != chunk_counts->tail; + chunk_c_node = chunk_c_node->next) { + if (is_sig_int_occurred) { + return SDAS_SIGINT; + } + fprintf(stderr, + "CHUNK %3" PRIu64 " of %3zu\n", + ++chunk_count, + chunk_counts->count); + // Write file count before iterating through files. + if (non_c_chunk_size) { + *non_c_chunk_size = 0; + } + + u64 = (*((uint64_t *)chunk_c_node->data)); + simple_archiver_helper_64_bit_be(&u64); + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + SDArchiverLLNode *saved_node = file_node; + for (uint64_t file_idx = 0; file_idx < *((uint64_t *)chunk_c_node->data); + ++file_idx) { + file_node = file_node->next; + if (file_node == files_list->tail) { + return SDAS_INTERNAL_ERROR; + } + const SDArchiverInternalFileInfo *file_info_struct = file_node->data; + if (non_c_chunk_size) { + *non_c_chunk_size += file_info_struct->file_size; + } + const size_t filename_len = strlen(file_info_struct->filename); + if (state->parsed->prefix) { + const size_t total_length = filename_len + prefix_length; + if (total_length >= 0xFFFF) { + fprintf(stderr, "ERROR: Filename with prefix is too large!\n"); + return SDAS_INVALID_FILE; + } + u16 = (uint16_t)total_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(state->parsed->prefix, 1, prefix_length, out_f) + != prefix_length) { + return SDAS_FAILED_TO_WRITE; + } else if (fwrite(file_info_struct->filename, + 1, + filename_len + 1, + out_f) + != filename_len + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + if (filename_len >= 0xFFFF) { + fprintf(stderr, "ERROR: Filename is too large!\n"); + return SDAS_INVALID_FILE; + } + u16 = (uint16_t)filename_len; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(file_info_struct->filename, 1, u16 + 1, out_f) != + (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + if (fwrite(file_info_struct->bit_flags, 1, 4, out_f) != 4) { + return SDAS_FAILED_TO_WRITE; + } + // UID and GID. + + // Forced UID/GID is already handled by "symlinks_and_files_from_files". + + u32 = file_info_struct->uid; + if ((state->parsed->flags & 0x400) == 0) { + uint32_t mapped_uid; + if (simple_archiver_get_uid_mapping(state->parsed->mappings, + state->parsed->users_infos, + u32, + &mapped_uid, + NULL) == 0) { + u32 = mapped_uid; + } + } + simple_archiver_helper_32_bit_be(&u32); + if (fwrite(&u32, 4, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + u32 = file_info_struct->gid; + if ((state->parsed->flags & 0x800) == 0) { + uint32_t mapped_gid; + if(simple_archiver_get_gid_mapping(state->parsed->mappings, + state->parsed->users_infos, + u32, + &mapped_gid, + NULL) == 0) { + u32 = mapped_gid; + } + } + simple_archiver_helper_32_bit_be(&u32); + if (fwrite(&u32, 4, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + u32 = file_info_struct->uid; + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *to_cleanup_user = NULL; + const char *username = simple_archiver_hash_map_get( + state->parsed->users_infos.UidToUname, &u32, sizeof(uint32_t)); + if (username) { + if ((state->parsed->flags & 0x400) == 0) { + uint32_t out_uid; + const char *mapped_username = NULL; + if (simple_archiver_get_user_mapping(state->parsed->mappings, + state->parsed->users_infos, + username, + &out_uid, + &mapped_username) == 0 + && mapped_username) { + username = mapped_username; + to_cleanup_user = (char *)mapped_username; + } + } + unsigned long name_length = strlen(username); + if (name_length > 0xFFFF) { + return SDAS_INTERNAL_ERROR; + } + u16 = (uint16_t)name_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(username, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + u32 = file_info_struct->gid; + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *to_cleanup_group = NULL; + const char *groupname = simple_archiver_hash_map_get( + state->parsed->users_infos.GidToGname, &u32, sizeof(uint32_t)); + if (groupname) { + if ((state->parsed->flags & 0x800) == 0) { + uint32_t out_gid; + const char *mapped_group = NULL; + if (simple_archiver_get_group_mapping(state->parsed->mappings, + state->parsed->users_infos, + groupname, + &out_gid, + &mapped_group) == 0 + && mapped_group) { + groupname = mapped_group; + to_cleanup_group = (char *)mapped_group; + } + } + unsigned long group_length = strlen(groupname); + if (group_length > 0xFFFF) { + return SDAS_INTERNAL_ERROR; + } + u16 = (uint16_t)group_length; + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + simple_archiver_helper_16_bit_be(&u16); + if (fwrite(groupname, 1, u16 + 1, out_f) != (size_t)u16 + 1) { + return SDAS_FAILED_TO_WRITE; + } + } else { + u16 = 0; + if (fwrite(&u16, 2, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + uint64_t u64 = file_info_struct->file_size; + simple_archiver_helper_64_bit_be(&u64); + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + } + + file_node = saved_node; + + if (state->parsed->compressor && state->parsed->decompressor) { + // Is compressing. + __attribute__((cleanup(simple_archiver_helper_cleanup_c_string))) + char *temp_filename = NULL; + + __attribute__((cleanup(cleanup_temp_filename_delete))) void **ptrs_array = + malloc(sizeof(void *) * 2); + ptrs_array[0] = NULL; + ptrs_array[1] = NULL; + + __attribute__((cleanup(simple_archiver_helper_cleanup_FILE))) + FILE *temp_fd = simple_archiver_helper_temp_dir(state->parsed, + &temp_filename); + ptrs_array[0] = temp_filename; + + if (!temp_fd) { + temp_fd = tmpfile(); + if (!temp_fd) { + fprintf(stderr, + "ERROR: Failed to create a temporary file for archival!\n"); + return SDAS_INTERNAL_ERROR; + } + } + + // Handle SIGPIPE. + is_sig_pipe_occurred = 0; + signal(SIGPIPE, handle_sig_pipe); + + int pipe_into_cmd[2]; + int pipe_outof_cmd[2]; + __attribute__((cleanup( + simple_archiver_internal_cleanup_decomp_pid))) pid_t compressor_pid = + -1; + + if (pipe(pipe_into_cmd) != 0) { + // Unable to create pipes. + return SDAS_INTERNAL_ERROR; + } else if (pipe(pipe_outof_cmd) != 0) { + // Unable to create second set of pipes. + close(pipe_into_cmd[0]); + close(pipe_into_cmd[1]); + return SDAS_INTERNAL_ERROR; + } else if (fcntl(pipe_into_cmd[1], F_SETFL, O_NONBLOCK) != 0) { + // Unable to set non-blocking on into-write-pipe. + close(pipe_into_cmd[0]); + close(pipe_into_cmd[1]); + close(pipe_outof_cmd[0]); + close(pipe_outof_cmd[1]); + return SDAS_INTERNAL_ERROR; + } else if (fcntl(pipe_outof_cmd[0], F_SETFL, O_NONBLOCK) != 0) { + // Unable to set non-blocking on outof-read-pipe. + close(pipe_into_cmd[0]); + close(pipe_into_cmd[1]); + close(pipe_outof_cmd[0]); + close(pipe_outof_cmd[1]); + return SDAS_INTERNAL_ERROR; + } else if (simple_archiver_de_compress(pipe_into_cmd, pipe_outof_cmd, + state->parsed->compressor, + &compressor_pid) != 0) { + // Failed to spawn compressor. + close(pipe_into_cmd[1]); + close(pipe_outof_cmd[0]); + fprintf(stderr, + "WARNING: Failed to start compressor cmd! Invalid cmd?\n"); + return SDAS_INTERNAL_ERROR; + } + + // Close unnecessary pipe fds on this end of the transfer. + close(pipe_into_cmd[0]); + close(pipe_outof_cmd[1]); + + // Set up cleanup so that remaining open pipes in this side is cleaned up. + __attribute__((cleanup( + simple_archiver_internal_cleanup_int_fd))) int pipe_outof_read = + pipe_outof_cmd[0]; + __attribute__((cleanup( + simple_archiver_internal_cleanup_int_fd))) int pipe_into_write = + pipe_into_cmd[1]; + + int_fast8_t to_temp_finished = 0; + for (uint64_t file_idx = 0; file_idx < *((uint64_t *)chunk_c_node->data); + ++file_idx) { + if (is_sig_int_occurred) { + return SDAS_SIGINT; + } + file_node = file_node->next; + if (file_node == files_list->tail) { + return SDAS_INTERNAL_ERROR; + } + const SDArchiverInternalFileInfo *file_info_struct = file_node->data; + fprintf(stderr, + " FILE %3" PRIu64 " of %3" PRIu64 ": %s\n", + file_idx + 1, + *(uint64_t *)chunk_c_node->data, + file_info_struct->filename); + __attribute__((cleanup(simple_archiver_helper_cleanup_FILE))) FILE *fd = + fopen(file_info_struct->filename, "rb"); + + int_fast8_t to_comp_finished = 0; + char hold_buf[SIMPLE_ARCHIVER_BUFFER_SIZE]; + ssize_t has_hold = -1; + while (!to_comp_finished) { + if (is_sig_pipe_occurred) { + fprintf(stderr, "ERROR: SIGPIPE while compressing!\n"); + return SDAS_INTERNAL_ERROR; + } + if (!to_comp_finished) { + // Write to compressor. + if (ferror(fd)) { + fprintf(stderr, "ERROR: Writing to chunk, file read error!\n"); + return SDAS_INTERNAL_ERROR; + } + if (has_hold < 0) { + size_t fread_ret = fread(buf, 1, SIMPLE_ARCHIVER_BUFFER_SIZE, fd); + if (fread_ret > 0) { + ssize_t write_ret = write(pipe_into_write, buf, fread_ret); + if (write_ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // Non-blocking write. + has_hold = (int)fread_ret; + memcpy(hold_buf, buf, fread_ret); + nanosleep(&nonblock_sleep, NULL); + } else { + fprintf( + stderr, + "ERROR: Writing to compressor, pipe write error!\n"); + return SDAS_FAILED_TO_WRITE; + } + } else if (write_ret == 0) { + fprintf( + stderr, + "ERROR: Writing to compressor, unable to write bytes!\n"); + return SDAS_FAILED_TO_WRITE; + } else if ((size_t)write_ret < fread_ret) { + has_hold = (ssize_t)fread_ret - write_ret; + memcpy(hold_buf, buf + write_ret, (size_t)has_hold); + } + } + + if (feof(fd) && has_hold < 0) { + to_comp_finished = 1; + } + } else { + ssize_t write_ret = + write(pipe_into_write, hold_buf, (size_t)has_hold); + if (write_ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // Non-blocking write. + nanosleep(&nonblock_sleep, NULL); + } else { + return SDAS_INTERNAL_ERROR; + } + } else if (write_ret < has_hold) { + memcpy(buf, hold_buf + write_ret, + (size_t)(has_hold - write_ret)); + memcpy(hold_buf, buf, (size_t)(has_hold - write_ret)); + has_hold = has_hold - write_ret; + } else if (write_ret != has_hold) { + return SDAS_INTERNAL_ERROR; + } else { + has_hold = -1; + } + } + } + + // Write compressed data to temp file. + ssize_t read_ret = + read(pipe_outof_read, buf, SIMPLE_ARCHIVER_BUFFER_SIZE); + if (read_ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // Non-blocking read. + nanosleep(&nonblock_sleep, NULL); + } else { + fprintf(stderr, + "ERROR: Reading from compressor, pipe read error!\n"); + return SDAS_INTERNAL_ERROR; + } + } else if (read_ret == 0) { + // EOF. + to_temp_finished = 1; + } else { + size_t fwrite_ret = fwrite(buf, 1, (size_t)read_ret, temp_fd); + if (fwrite_ret != (size_t)read_ret) { + fprintf(stderr, + "ERROR: Reading from compressor, failed to write to " + "temporary file!\n"); + return SDAS_INTERNAL_ERROR; + } + } + } + } + + simple_archiver_internal_cleanup_int_fd(&pipe_into_write); + + // Finish writing. + if (!to_temp_finished) { + while (1) { + if (is_sig_pipe_occurred) { + fprintf(stderr, "ERROR: SIGPIPE while compressing!\n"); + return SDAS_INTERNAL_ERROR; + } + ssize_t read_ret = + read(pipe_outof_read, buf, SIMPLE_ARCHIVER_BUFFER_SIZE); + if (read_ret < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + // Non-blocking read. + nanosleep(&nonblock_sleep, NULL); + } else { + fprintf(stderr, + "ERROR: Reading from compressor, pipe read error!\n"); + return SDAS_INTERNAL_ERROR; + } + } else if (read_ret == 0) { + // EOF. + break; + } else { + size_t fwrite_ret = fwrite(buf, 1, (size_t)read_ret, temp_fd); + if (fwrite_ret != (size_t)read_ret) { + fprintf(stderr, + "ERROR: Reading from compressor, failed to write to " + "temporary file!\n"); + return SDAS_INTERNAL_ERROR; + } + } + } + } + + long comp_chunk_size = ftell(temp_fd); + if (comp_chunk_size < 0) { + fprintf(stderr, + "ERROR: Temp file reported negative size after compression!\n"); + return SDAS_INTERNAL_ERROR; + } + + // Write compressed chunk size. + uint64_t u64 = (uint64_t)comp_chunk_size; + simple_archiver_helper_64_bit_be(&u64); + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + rewind(temp_fd); + + size_t written_size = 0; + + // Write compressed chunk. + while (!feof(temp_fd)) { + if (is_sig_int_occurred) { + return SDAS_SIGINT; + } else if (ferror(temp_fd)) { + return SDAS_INTERNAL_ERROR; + } + size_t fread_ret = fread(buf, 1, SIMPLE_ARCHIVER_BUFFER_SIZE, temp_fd); + if (fread_ret > 0) { + size_t fwrite_ret = fwrite(buf, 1, fread_ret, out_f); + written_size += fread_ret; + if (fwrite_ret != fread_ret) { + fprintf(stderr, + "ERROR: Partial write of read bytes from temp file to " + "output file!\n"); + return SDAS_FAILED_TO_WRITE; + } + } + } + + if (written_size != (size_t)comp_chunk_size) { + fprintf(stderr, + "ERROR: Written chunk size is not actual chunk size!\n"); + return SDAS_FAILED_TO_WRITE; + } + + // Cleanup and remove temp_fd. + simple_archiver_helper_cleanup_FILE(&temp_fd); + } else { + // Is NOT compressing. + if (!non_c_chunk_size) { + return SDAS_INTERNAL_ERROR; + } + simple_archiver_helper_64_bit_be(non_c_chunk_size); + fwrite(non_c_chunk_size, 8, 1, out_f); + for (uint64_t file_idx = 0; file_idx < *((uint64_t *)chunk_c_node->data); + ++file_idx) { + if (is_sig_int_occurred) { + return SDAS_SIGINT; + } + file_node = file_node->next; + if (file_node == files_list->tail) { + return SDAS_INTERNAL_ERROR; + } + const SDArchiverInternalFileInfo *file_info_struct = file_node->data; + fprintf(stderr, + " FILE %3" PRIu64 " of %3" PRIu64 ": %s\n", + file_idx + 1, + *(uint64_t *)chunk_c_node->data, + file_info_struct->filename); + __attribute__((cleanup(simple_archiver_helper_cleanup_FILE))) FILE *fd = + fopen(file_info_struct->filename, "rb"); + while (!feof(fd)) { + if (ferror(fd)) { + fprintf(stderr, "ERROR: Writing to chunk, file read error!\n"); + return SDAS_INTERNAL_ERROR; + } + size_t fread_ret = fread(buf, 1, SIMPLE_ARCHIVER_BUFFER_SIZE, fd); + if (fread_ret > 0) { + size_t fwrite_ret = fwrite(buf, 1, fread_ret, out_f); + if (fwrite_ret != fread_ret) { + fprintf(stderr, "ERROR: Writing to chunk, file write error!\n"); + return SDAS_FAILED_TO_WRITE; + } + } + } + } + } + } + + // Write directory entries. + + if (dirs_list->count > 0xFFFFFFFF) { + return SDAS_TOO_MANY_DIRS; + } + + u64 = dirs_list->count; + if (u64 != 0) { + fprintf(stderr, "Directories:\n"); + } + + simple_archiver_helper_64_bit_be(&u64); + + if (fwrite(&u64, 8, 1, out_f) != 1) { + return SDAS_FAILED_TO_WRITE; + } + + void **void_ptrs = malloc(sizeof(void*) * 2); + void_ptrs[0] = out_f; + void_ptrs[1] = state; + + if (simple_archiver_list_get(dirs_list, + internal_write_dir_entries_v2_v3_v4, + void_ptrs)) { + free(void_ptrs); + return SDAS_INTERNAL_ERROR; + } + free(void_ptrs); + + return SDAS_SUCCESS; +} + int simple_archiver_parse_archive_info(FILE *in_f, int_fast8_t do_extract, const SDArchiverState *state) { #if SIMPLE_ARCHIVER_PLATFORM == SIMPLE_ARCHIVER_PLATFORM_COSMOPOLITAN || \ -- 2.49.0