Merge branch 'issue_18_improve_compression'

Resolves #18

CMakeLists.txt

@@ -28,6 +28,49 @@ endif()
 
 add_executable(simplearchiver ${SimpleArchiver_SOURCES})
 
+target_compile_options(simplearchiver PUBLIC
+    -Wall -Wformat -Wformat=2 -Wconversion -Wimplicit-fallthrough
+    -Werror=format-security
+    -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3
+    -D_GLIBCXX_ASSERTIONS
+    -fstrict-flex-arrays=3
+    -fstack-clash-protection -fstack-protector-strong
+    -Wl,-z,nodlopen -Wl,-z,noexecstack
+    -Wl,-z,relro -Wl,-z,now
+    -Wl,--as-needed -Wl,--no-copy-dt-needed-entries
+    -fPIE -pie
+)
+
+target_link_options(simplearchiver PUBLIC
+    -Wall -Wformat -Wformat=2 -Wconversion -Wimplicit-fallthrough
+    -Werror=format-security
+    -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3
+    -D_GLIBCXX_ASSERTIONS
+    -fstrict-flex-arrays=3
+    -fstack-clash-protection -fstack-protector-strong
+    -Wl,-z,nodlopen -Wl,-z,noexecstack
+    -Wl,-z,relro -Wl,-z,now
+    -Wl,--as-needed -Wl,--no-copy-dt-needed-entries
+    -fPIE -pie
+)
+
+# Inhibit format-string-related warning in src/archiver.c .
+set_source_files_properties(src/archiver.c
+    PROPERTIES
+        COMPILE_FLAGS -Wno-format-nonliteral
+)
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+    target_compile_options(simplearchiver PUBLIC
+        -fno-delete-null-pointer-checks -fno-strict-overflow
+        -fno-strict-aliasing -ftrivial-auto-var-init=zero
+    )
+    target_link_options(simplearchiver PUBLIC
+        -fno-delete-null-pointer-checks -fno-strict-overflow
+        -fno-strict-aliasing -ftrivial-auto-var-init=zero
+    )
+endif()
+
 add_executable(test_datastructures
     src/data_structures/test.c
     src/data_structures/linked_list.c
@@ -40,6 +83,7 @@ add_executable(test_simplearchiver
     src/test.c
     src/parser.c
     src/helpers.c
+    src/archiver.c
     src/algorithms/linear_congruential_gen.c
     src/data_structures/linked_list.c
     src/data_structures/hash_map.c

README.md

@@ -20,13 +20,15 @@ API calls.
       Use "-f -" to work on stdout when creating archive or stdin when reading archive
       NOTICE: "-f" is not affected by "-C"!
     -C <dir> : Change current working directory before archiving/extracting
-    --compressor <full_compress_cmd> : requires --decompressor
+    --compressor <full_compress_cmd> : requires --decompressor and cmd must use stdin/stdout
-    --decompressor <full_decompress_cmd> : requires --compressor
+    --decompressor <full_decompress_cmd> : requires --compressor and cmd must use stdin/stdout
       Specifying "--decompressor" when extracting overrides archive file's stored decompressor cmd
     --overwrite-create : allows overwriting an archive file
     --overwrite-extract : allows overwriting when extracting
     --no-abs-symlink : do not store absolute paths for symlinks
     --temp-files-dir <dir> : where to store temporary files created when compressing (defaults to current working directory)
+    --write-version <version> : Force write version file format (default 1)
+    --chunk-min-size <bytes> : v1 file format minimum chunk size (default 4194304 or 4MiB)
     -- : specifies remaining arguments are files to archive/extract
     If creating archive file, remaining args specify files to archive.
     If extracting archive file, remaining args specify files to extract.

file_format.md

@@ -76,3 +76,117 @@ Following the file-count bytes, the following bytes are added for each file:
     1. 8 bytes 64-bit unsigned integer "size of filename in this archive file"
        in big-endian.
     2. X bytes file data (length defined by previous value).
+
+## Format Version 1
+
+File extension is "*.simplearchive" but this isn't really checked.
+
+First 18 bytes of file will be (in ascii):
+
+    SIMPLE_ARCHIVE_VER
+
+Next 2 bytes is a 16-bit unsigned integer "version" in big-endian. It will be:
+
+    0x00 0x01
+
+Next 4 bytes are bit-flags.
+
+1. The first byte
+    1. The first bit is set if de/compressor is set for this archive.
+
+The remaining unused flags in the previous bit-flags bytes are reserved for
+future revisions and are currently ignored.
+
+If the previous "de/compressor is set" flag is enabled, then the next section is
+added:
+
+1. 2 bytes is 16-bit unsigned integer "compressor cmd+args" in big-endian. This
+   does not include the NULL at the end of the string.
+2. X bytes of "compressor cmd+args" (length defined by previous value). Is a
+   NULL-terminated string.
+3. 2 bytes is 16-bit unsigned integer "decompressor cmd+args" in big-endian.
+   This does not include the NULL at the end of the string.
+4. X bytes of "decompressor cmd+args" (length defined by previous value). Is a
+   NULL-terminated string.
+
+The next 4 bytes is a 32-bit unsigned integer "link count" in big-endian which
+will indicate the number of symbolic links in this archive.
+
+Following the link-count bytes, the following bytes are added for each symlink:
+
+1. 2 bytes bit-flags:
+    1. The first byte.
+        1. The first bit is UNSET if relative links are preferred, and is SET if
+           absolute links are preferred.
+        2. The second bit is "user read permission".
+        3. The third bit is "user write permission".
+        4. The fourth bit is "user execute permission".
+        5. The fifth bit is "group read permission".
+        6. The sixth bit is "group write permission".
+        7. The seventh bit is "group execute permission".
+        8. The eighth bit is "other read permission".
+    2. The second byte.
+        1. The first bit is "other write permission".
+        2. The second bit is "other execute permission".
+2. 2 bytes 16-bit unsigned integer "link name" in big-endian. This does not
+   include the NULL at the end of the string. Must not be zero.
+3. X bytes of link-name (length defined by previous value). Is a NULL-terminated
+   string.
+4. 2 bytes is 16-bit unsigned integer "link target absolute path" in
+   big-endian. This does not include the NULL at the end of the string.
+5. X bytes of link-target-absolute-path (length defined by previous value).
+   Is a NULL-terminated string. If the previous "size" value is 0, then
+   this entry does not exist and should be skipped.
+6. 2 bytes is 16-bit unsigned integer "link target relative path" in
+   big-endian. This does not include the NULL at the end of the string.
+7. X bytes of link-target-relative-path (length defined by previous value).
+   Is a NULL-terminated string. If the previous "size" value is 0, then
+   this entry does not exist and should be skipped.
+
+After the symlink related data, the next 4 bytes is a 32-bit unsigned integer
+"chunk count" in big-endian which will indicate the number of chunks in this
+archive.
+
+Following the chunk-count bytes, the following bytes are added for each chunk:
+
+1. 4 bytes that are a 32-bit unsigned integer "file count" in big-endian.
+
+The following bytes are added for each file within the current chunk:
+
+1. 2 bytes that are a 16-bit unsigned integer "filename length" in big-endian.
+   This does not include the NULL at the end of the string.
+2. X bytes of filename (length defined by previous value). Is a NULL-terminated
+   string.
+3. 4 bytes bit-flags.
+    1. The first byte.
+        1. The first bit is "user read permission".
+        2. The second bit is "user write permission".
+        3. The third bit is "user execute permission".
+        4. The fourth bit is "group read permission".
+        5. The fifth bit is "group write permission".
+        6. The sixth bit is "group execute permission".
+        7. The seventh bit is "other read permission".
+        8. The eighth bit is "other write permission".
+    2. The second byte.
+        1. The first bit is "other execute permission".
+    3. The third byte.
+        1. Currently unused.
+    4. The fourth byte.
+        1. Currently unused.
+4. Two 4-byte unsigned integers in big-endian for UID and GID.
+    1. A 32-bit unsigned integer in big endian that specifies the UID of the
+       file. Note that during extraction, if the user is not root, then this
+       value will be ignored.
+    2. A 32-bit unsigned integer in big endian that specifies the GID of the
+       file. Note that during extraction, if the user is not root, then this
+       value will be ignored.
+5. A 64-bit unsigned integer in big endian for the "size of file".
+
+After the files' metadata are the current chunk's data:
+
+1. A 64-bit unsigned integer in big endian for the "size of chunk".
+2. X bytes of data for the current chunk of the previously specified size. If
+   not using de/compressor, this section is the previously mentioned files
+   concatenated with each other. If using de/compressor, this section is the
+   previously mentioned files concatenated and compressed into a single blob of
+   data.

src/archiver.h

@@ -40,7 +40,7 @@ typedef struct SDArchiverState {
   size_t digits;
 } SDArchiverState;
 
-enum SDArchiverStateReturns {
+typedef enum SDArchiverStateReturns {
   SDAS_SUCCESS = 0,
   SDAS_HEADER_ALREADY_WRITTEN = 1,
   SDAS_FAILED_TO_WRITE,
@@ -51,8 +51,9 @@ enum SDArchiverStateReturns {
   SDAS_INTERNAL_ERROR,
   SDAS_FAILED_TO_CREATE_MAP,
   SDAS_FAILED_TO_EXTRACT_SYMLINK,
-  SDAS_FAILED_TO_CHANGE_CWD
+  SDAS_FAILED_TO_CHANGE_CWD,
-};
+  SDAS_INVALID_WRITE_VERSION
+} SDArchiverStateReturns;
 
 /// Returned pointer must not be freed.
 char *simple_archiver_error_to_string(enum SDArchiverStateReturns error);
@@ -65,12 +66,46 @@ void simple_archiver_free_state(SDArchiverState **state);
 int simple_archiver_write_all(FILE *out_f, SDArchiverState *state,
                               const SDArchiverLinkedList *filenames);
 
+int simple_archiver_write_v0(FILE *out_f, SDArchiverState *state,
+                             const SDArchiverLinkedList *filenames);
+
+int simple_archiver_write_v1(FILE *out_f, SDArchiverState *state,
+                             const SDArchiverLinkedList *filenames);
+
 /// Returns zero on success.
 int simple_archiver_parse_archive_info(FILE *in_f, int_fast8_t do_extract,
                                        const SDArchiverState *state);
 
+/// Returns zero on success.
+int simple_archiver_parse_archive_version_0(FILE *in_f, int_fast8_t do_extract,
+                                            const SDArchiverState *state);
+
+/// Returns zero on success.
+int simple_archiver_parse_archive_version_1(FILE *in_f, int_fast8_t do_extract,
+                                            const SDArchiverState *state);
+
 /// Returns zero on success.
 int simple_archiver_de_compress(int pipe_fd_in[2], int pipe_fd_out[2],
                                 const char *cmd, void *pid_out);
 
+/// If returns non-NULL, must be free'd.
+char *simple_archiver_filenames_to_relative_path(const char *from_abs,
+                                                 const char *to_abs);
+
+/// Gets the absolute path to a file given a path to a file.
+/// Should also work on symlinks such that the returned string is the path to
+/// the link itself, not what it points to.
+/// Non-NULL on success, and must be free'd if non-NULL.
+char *simple_archiver_file_abs_path(const char *filename);
+
+/// Used to validate a file in a ".simplearchive" file to avoid writing outside
+/// of current working directory.
+/// Returns zero if file is OK.
+/// Returns 1 if file starts with '/'.
+/// Returns 2 if file contains '../' at the start.
+/// Returns 3 if file contains '/../' in the middle.
+/// Returns 4 if file contains '/..' at the end.
+/// Returns 5 if "filepath" is NULL.
+int simple_archiver_validate_file_path(const char *filepath);
+
 #endif

src/main.c

@@ -67,12 +67,13 @@ int main(int argc, const char **argv) {
     }
   }
 
+  SDArchiverParsedStatus parsed_status;
   __attribute__((cleanup(simple_archiver_list_free)))
   SDArchiverLinkedList *filenames =
-      simple_archiver_parsed_to_filenames(&parsed);
+      simple_archiver_parsed_to_filenames(&parsed, &parsed_status);
-  if (!filenames) {
+  if (!filenames || parsed_status != SDAPS_SUCCESS) {
-    fprintf(stderr,
+    fprintf(stderr, "ERROR: %s!\n",
-            "ERROR: Failed to resolve filenames from positional arguments!\n");
+            simple_archiver_parsed_status_to_str(parsed_status));
     return 8;
   }
 
@@ -96,7 +97,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_write_all(file, state, filenames);
       if (ret != SDAS_SUCCESS) {
         fprintf(stderr, "Error during writing.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
       fclose(file);
@@ -112,7 +114,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_write_all(stdout, state, filenames);
       if (ret != SDAS_SUCCESS) {
         fprintf(stderr, "Error during writing.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
     }
@@ -129,7 +132,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_parse_archive_info(file, 0, NULL);
       if (ret != 0) {
         fprintf(stderr, "Error during archive checking/examining.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
       fclose(file);
@@ -137,7 +141,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_parse_archive_info(stdin, 0, NULL);
       if (ret != 0) {
         fprintf(stderr, "Error during archive checking/examining.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
     }
@@ -156,7 +161,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_parse_archive_info(file, 1, state);
       if (ret != SDAS_SUCCESS) {
         fprintf(stderr, "Error during archive extracting.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
       fclose(file);
@@ -164,7 +170,8 @@ int main(int argc, const char **argv) {
       int ret = simple_archiver_parse_archive_info(stdin, 1, state);
       if (ret != SDAS_SUCCESS) {
         fprintf(stderr, "Error during archive extracting.\n");
-        char *error_str = simple_archiver_error_to_string(ret);
+        char *error_str =
+            simple_archiver_error_to_string((SDArchiverStateReturns)ret);
         fprintf(stderr, "  %s\n", error_str);
       }
     }

src/parser.c

@@ -131,6 +131,17 @@ int list_remove_same_str_fn(void *data, void *ud) {
   return 0;
 }
 
+char *simple_archiver_parsed_status_to_str(SDArchiverParsedStatus status) {
+  switch (status) {
+    case SDAPS_SUCCESS:
+      return "Success";
+    case SDAPS_NO_USER_CWD:
+      return "No user current working directory (-C <dir>)";
+    default:
+      return "Unknown error";
+  }
+}
+
 void simple_archiver_print_usage(void) {
   fprintf(stderr, "Usage flags:\n");
   fprintf(stderr, "-c : create archive file\n");
@@ -145,9 +156,11 @@ void simple_archiver_print_usage(void) {
           "-C <dir> : Change current working directory before "
           "archiving/extracting\n");
   fprintf(stderr,
-          "--compressor <full_compress_cmd> : requires --decompressor\n");
+          "--compressor <full_compress_cmd> : requires --decompressor and cmd "
+          "must use stdin/stdout\n");
   fprintf(stderr,
-          "--decompressor <full_decompress_cmd> : requires --compressor\n");
+          "--decompressor <full_decompress_cmd> : requires --compressor and "
+          "cmd must use stdin/stdout\n");
   fprintf(stderr,
           "  Specifying \"--decompressor\" when extracting overrides archive "
           "file's stored decompressor cmd\n");
@@ -158,6 +171,12 @@ void simple_archiver_print_usage(void) {
   fprintf(stderr,
           "--temp-files-dir <dir> : where to store temporary files created "
           "when compressing (defaults to current working directory)\n");
+  fprintf(stderr,
+          "--write-version <version> : Force write version file format "
+          "(default 1)\n");
+  fprintf(stderr,
+          "--chunk-min-size <bytes> : v1 file format minimum chunk size "
+          "(default 4194304 or 4MiB)\n");
   fprintf(stderr,
           "-- : specifies remaining arguments are files to archive/extract\n");
   fprintf(
@@ -178,6 +197,8 @@ SDArchiverParsed simple_archiver_create_parsed(void) {
   parsed.working_files = NULL;
   parsed.temp_dir = NULL;
   parsed.user_cwd = NULL;
+  parsed.write_version = 1;
+  parsed.minimum_chunk_size = 4194304;
 
   return parsed;
 }
@@ -288,11 +309,50 @@ int simple_archiver_parse_args(int argc, const char **argv,
         out->temp_dir = argv[1];
         --argc;
         ++argv;
+      } else if (strcmp(argv[0], "--write-version") == 0) {
+        if (argc < 2) {
+          fprintf(stderr,
+                  "ERROR: --write-version expects an integer argument!\n");
+          simple_archiver_print_usage();
+          return 1;
+        }
+        int version = atoi(argv[1]);
+        if (version < 0) {
+          fprintf(stderr, "ERROR: --write-version cannot be negative!\n");
+          simple_archiver_print_usage();
+          return 1;
+        } else if (version > 1) {
+          fprintf(stderr, "ERROR: --write-version must be 0 or 1!\n");
+          simple_archiver_print_usage();
+          return 1;
+        }
+        out->write_version = (uint32_t)version;
+        --argc;
+        ++argv;
+      } else if (strcmp(argv[0], "--chunk-min-size") == 0) {
+        if (argc < 2) {
+          fprintf(stderr,
+                  "ERROR: --chunk-min-size expects an integer argument!\n");
+          simple_archiver_print_usage();
+          return 1;
+        }
+        out->minimum_chunk_size = strtoull(argv[1], NULL, 10);
+        if (out->minimum_chunk_size == 0) {
+          fprintf(stderr, "ERROR: --chunk-min-size cannot be zero!\n");
+          simple_archiver_print_usage();
+          return 1;
+        }
+        --argc;
+        ++argv;
       } else if (argv[0][0] == '-' && argv[0][1] == '-' && argv[0][2] == 0) {
         is_remaining_args = 1;
       } else if (argv[0][0] != '-') {
         is_remaining_args = 1;
         continue;
+      } else {
+        fprintf(stderr, "ERROR: Got invalid arg \"%s\"!\n", argv[0]);
+        simple_archiver_print_usage();
+        return 1;
       }
     } else {
       if (out->working_files == NULL) {
@@ -367,7 +427,7 @@ void simple_archiver_free_parsed(SDArchiverParsed *parsed) {
 }
 
 SDArchiverLinkedList *simple_archiver_parsed_to_filenames(
-    const SDArchiverParsed *parsed) {
+    const SDArchiverParsed *parsed, SDArchiverParsedStatus *status_out) {
   SDArchiverLinkedList *files_list = simple_archiver_list_init();
   __attribute__((cleanup(simple_archiver_hash_map_free)))
   SDArchiverHashMap *hash_map = simple_archiver_hash_map_init();
@@ -381,6 +441,9 @@ SDArchiverLinkedList *simple_archiver_parsed_to_filenames(
     original_cwd = realpath(".", NULL);
     if (chdir(parsed->user_cwd)) {
       simple_archiver_list_free(&files_list);
+      if (status_out) {
+        *status_out = SDAPS_NO_USER_CWD;
+      }
       return NULL;
     }
   }
@@ -607,5 +670,8 @@ SDArchiverLinkedList *simple_archiver_parsed_to_filenames(
     }
   }
 
+  if (status_out) {
+    *status_out = SDAPS_SUCCESS;
+  }
   return files_list;
 }

src/parser.h

@@ -51,6 +51,10 @@ typedef struct SDArchiverParsed {
   const char *temp_dir;
   /// Dir specified by "-C".
   const char *user_cwd;
+  /// Currently only 0 and 1 is supported.
+  uint32_t write_version;
+  /// The minimum size of a chunk in bytes (the last chunk may be less).
+  uint64_t minimum_chunk_size;
 } SDArchiverParsed;
 
 typedef struct SDArchiverFileInfo {
@@ -59,6 +63,14 @@ typedef struct SDArchiverFileInfo {
   char *link_dest;
 } SDArchiverFileInfo;
 
+typedef enum SDArchiverParsedStatus {
+  SDAPS_SUCCESS,
+  SDAPS_NO_USER_CWD,
+} SDArchiverParsedStatus;
+
+/// Returned c-string does not need to be free'd.
+char *simple_archiver_parsed_status_to_str(SDArchiverParsedStatus status);
+
 void simple_archiver_print_usage(void);
 
 SDArchiverParsed simple_archiver_create_parsed(void);
@@ -74,6 +86,6 @@ void simple_archiver_free_parsed(SDArchiverParsed *parsed);
 
 /// Each entry in the linked list is an SDArchiverFileInfo object.
 SDArchiverLinkedList *simple_archiver_parsed_to_filenames(
-    const SDArchiverParsed *parsed);
+    const SDArchiverParsed *parsed, SDArchiverParsedStatus *status_out);
 
 #endif

src/test.c

@@ -23,6 +23,7 @@
 #include <string.h>
 
 // Local includes.
+#include "archiver.h"
 #include "helpers.h"
 #include "parser_internal.h"
 
@@ -241,6 +242,27 @@ int main(void) {
     free(out);
   }
 
+  // Test archiver.
+  {
+    __attribute__((
+        cleanup(simple_archiver_helper_cleanup_c_string))) char *rel_path =
+        simple_archiver_filenames_to_relative_path(
+            "/one/two/three/four/five", "/one/two/branch/other/path");
+    CHECK_STREQ(rel_path, "../../branch/other/path");
+    simple_archiver_helper_cleanup_c_string(&rel_path);
+
+    rel_path = simple_archiver_filenames_to_relative_path(
+        "/one/two/three/four/five", "/one/two/three/other/dir/");
+    CHECK_STREQ(rel_path, "../other/dir/");
+    simple_archiver_helper_cleanup_c_string(&rel_path);
+
+    CHECK_FALSE(simple_archiver_validate_file_path("Local/Path"));
+    CHECK_TRUE(simple_archiver_validate_file_path("/Abs/Path"));
+    CHECK_TRUE(simple_archiver_validate_file_path("Local/../../not/really"));
+    CHECK_TRUE(simple_archiver_validate_file_path("./../almost"));
+    CHECK_TRUE(simple_archiver_validate_file_path("strange/.."));
+  }
+
   printf("Checks checked: %u\n", checks_checked);
   printf("Checks passed:  %u\n", checks_passed);
   return checks_passed == checks_checked ? 0 : 1;