#define _POSIX_C_SOURCE 200809L #include "pack.h" #include "verify.h" #include "scan.h" #include "deps.h" #include "provides.h" #include "package.h" #include "stage.h" #include #include #include #include #include #include #include #include #include #include #include #include #include struct tree_file { char *path; struct stat state; size_t anchor, members; }; struct writer { struct archive *archive; FILE *manifest; struct tree_file *files; size_t count, capacity, seen; int collecting; }; static int file_order(const void *left, const void *right) { const struct tree_file *a = left, *b = right; if (a->state.st_dev != b->state.st_dev) return a->state.st_dev < b->state.st_dev ? -1 : 1; if (a->state.st_ino != b->state.st_ino) return a->state.st_ino < b->state.st_ino ? -1 : 1; return strcmp(a->path, b->path); } static int unchanged(const struct stat *a, const struct stat *b) { return a->st_dev == b->st_dev && a->st_ino == b->st_ino && a->st_mode == b->st_mode && a->st_uid == b->st_uid && a->st_gid == b->st_gid && a->st_size == b->st_size && a->st_nlink == b->st_nlink && a->st_mtim.tv_sec == b->st_mtim.tv_sec && a->st_mtim.tv_nsec == b->st_mtim.tv_nsec && a->st_ctim.tv_sec == b->st_ctim.tv_sec && a->st_ctim.tv_nsec == b->st_ctim.tv_nsec; } static int remember_file(struct writer *writer, const char *path, const struct stat *st) { struct tree_file *file; if (writer->count == writer->capacity) { size_t capacity = writer->capacity ? writer->capacity * 2 : 32; void *next; if (capacity < writer->capacity || capacity > SIZE_MAX / sizeof *file) return 0; next = realloc(writer->files, capacity * sizeof *file); if (!next) return 0; writer->files = next; writer->capacity = capacity; } file = &writer->files[writer->count]; memset(file, 0, sizeof *file); file->path = strdup(path); file->state = *st; if (!file->path) return 0; ++writer->count; return 1; } static void free_files(struct writer *writer) { size_t i; for (i = 0; i < writer->count; ++i) free(writer->files[i].path); free(writer->files); } static int group_id(const char *path, char output[65]) { unsigned char hash[32]; unsigned size; size_t i; if (EVP_Digest(path, strlen(path), hash, &size, EVP_sha256(), NULL) != 1 || size != 32) return 0; for (i = 0; i < 32; ++i) snprintf(output + i * 2, 3, "%02x", hash[i]); return 1; } static int write_manifest_path(FILE *file, const char *path) { const unsigned char *p = (const unsigned char *)path; if (fputc('"', file) == EOF) return 0; for (; *p; ++p) { if (*p == '"' || *p == '\\') { if (fputc('\\', file) == EOF || fputc(*p, file) == EOF) return 0; } else if (*p < 0x21 || *p >= 0x7f) { if (fprintf(file, "\\x%02x", (unsigned)*p) < 0) return 0; } else if (fputc(*p, file) == EOF) return 0; } return fputc('"', file) != EOF; } static int write_entry(struct writer *writer, int parent, const char *name, const char *archive_path, int directory) { struct archive_entry *entry = NULL; struct stat st, original; char buffer[65536]; EVP_MD_CTX *hash = NULL; unsigned char digest[32]; unsigned int digest_size; int fd = -1, ok = 0; const char *target = NULL; char group[65] = "-"; ssize_t got; fd = openat(parent, name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC | (directory ? O_DIRECTORY : O_NONBLOCK)); if (fd < 0 || fstat(fd, &st) || (directory ? !S_ISDIR(st.st_mode) : !S_ISREG(st.st_mode)) || (!directory && (st.st_size < 0 || (strncmp(archive_path, "DATA/", 5) && st.st_nlink != 1))) || flistxattr(fd, NULL, 0) != 0) goto done; original = st; if (writer->collecting) { ok = directory || remember_file(writer, archive_path, &st); goto done; } if (!directory && !strncmp(archive_path, "DATA/", 5)) { struct tree_file key, *file; memset(&key, 0, sizeof key); key.path = (char *)archive_path; key.state = st; file = writer->count ? bsearch(&key, writer->files, writer->count, sizeof *file, file_order) : NULL; if (!file || !unchanged(&file->state, &st)) goto done; ++writer->seen; if (file->members > 1) { const char *anchor = writer->files[file->anchor].path; if (!group_id(anchor, group)) goto done; if (strcmp(anchor, archive_path)) target = anchor; } } if (writer->archive) { entry = archive_entry_new(); if (!entry) goto done; archive_entry_set_pathname(entry, archive_path); archive_entry_set_filetype(entry, directory ? AE_IFDIR : AE_IFREG); archive_entry_set_perm(entry, st.st_mode & 07777); archive_entry_set_uid(entry, st.st_uid); archive_entry_set_gid(entry, st.st_gid); archive_entry_set_mtime(entry, 0, 0); archive_entry_set_size(entry, directory || target ? 0 : st.st_size); if (target) archive_entry_set_hardlink(entry, target); if (archive_write_header(writer->archive, entry) != ARCHIVE_OK) goto done; } if (writer->manifest && !directory) { hash = EVP_MD_CTX_new(); if (!hash || EVP_DigestInit_ex(hash, EVP_sha256(), NULL) != 1) goto done; } if (!directory) { off_t offset = 0; while (offset < st.st_size) { size_t want = st.st_size - offset < (off_t)sizeof buffer ? (size_t)(st.st_size - offset) : sizeof buffer; got = read(fd, buffer, want); if (got < 0 && errno == EINTR) continue; if (got <= 0 || (writer->archive && !target && archive_write_data(writer->archive, buffer, (size_t)got) != got) || (hash && EVP_DigestUpdate(hash, buffer, (size_t)got) != 1)) goto done; offset += got; } if (fstat(fd, &st) || st.st_size != offset || !unchanged(&original, &st)) goto done; if (hash && (EVP_DigestFinal_ex(hash, digest, &digest_size) != 1 || digest_size != sizeof digest)) goto done; } if (writer->manifest) { size_t i; if (fprintf(writer->manifest, "%s ", directory ? "dir" : target ? "hardlink" : "file") < 0 || !write_manifest_path(writer->manifest, archive_path + 5) || fprintf(writer->manifest, " %o - - %lu %lu %lld ", (unsigned)(st.st_mode & 07777), (unsigned long)st.st_uid, (unsigned long)st.st_gid, directory ? 0LL : (long long)st.st_size) < 0) goto done; if (directory) { if (fputs("-", writer->manifest) == EOF) goto done; } else for (i = 0; i < sizeof digest; ++i) if (fprintf(writer->manifest, "%02x", (unsigned)digest[i]) < 0) goto done; if (fprintf(writer->manifest, " none - %s", group) < 0 || (target && (fputc(' ', writer->manifest) == EOF || !write_manifest_path(writer->manifest, target + 5))) || fputc('\n', writer->manifest) == EOF) goto done; } ok = 1; done: archive_entry_free(entry); EVP_MD_CTX_free(hash); if (fd >= 0) close(fd); if (!ok) fprintf(stderr, "holypkg: cannot pack %s: %s\n", archive_path, writer->archive && archive_error_string(writer->archive) ? archive_error_string(writer->archive) : "unsupported input"); return ok; } static int write_symlink(struct writer *writer, int parent, const char *name, const char *archive_path) { struct stat before, after; struct archive_entry *entry = NULL; char *target = NULL, *procpath = NULL; size_t capacity = 128, length = strlen(name); ssize_t got; int ok = 0; if (fstatat(parent, name, &before, AT_SYMLINK_NOFOLLOW) || !S_ISLNK(before.st_mode) || before.st_nlink != 1 || length > SIZE_MAX - 64) goto done; procpath = malloc(length + 64); if (!procpath) goto done; snprintf(procpath, length + 64, "/proc/self/fd/%d/%s", parent, name); if (llistxattr(procpath, NULL, 0) != 0) goto done; for (;;) { char *grown = realloc(target, capacity); if (!grown) goto done; target = grown; got = readlinkat(parent, name, target, capacity - 1); if (got < 0) goto done; if ((size_t)got < capacity - 1) break; if (capacity > SIZE_MAX / 2) goto done; capacity *= 2; } target[got] = '\0'; if (!holy_safe_link(archive_path + 5, target) || fstatat(parent, name, &after, AT_SYMLINK_NOFOLLOW) || before.st_dev != after.st_dev || before.st_ino != after.st_ino || before.st_mode != after.st_mode || before.st_nlink != after.st_nlink || before.st_uid != after.st_uid || before.st_gid != after.st_gid || before.st_ctim.tv_sec != after.st_ctim.tv_sec || before.st_ctim.tv_nsec != after.st_ctim.tv_nsec) goto done; if (writer->archive) { entry = archive_entry_new(); if (!entry) goto done; archive_entry_set_pathname(entry, archive_path); archive_entry_set_filetype(entry, AE_IFLNK); archive_entry_set_perm(entry, before.st_mode & 07777); archive_entry_set_uid(entry, before.st_uid); archive_entry_set_gid(entry, before.st_gid); archive_entry_set_mtime(entry, 0, 0); archive_entry_set_size(entry, 0); archive_entry_set_symlink(entry, target); if (archive_write_header(writer->archive, entry) != ARCHIVE_OK) goto done; } if (writer->manifest && (fputs("symlink ", writer->manifest) == EOF || !write_manifest_path(writer->manifest, archive_path + 5) || fprintf(writer->manifest, " %o - - %lu %lu 0 - none - - ", (unsigned)(before.st_mode & 07777), (unsigned long)before.st_uid, (unsigned long)before.st_gid) < 0 || !write_manifest_path(writer->manifest, target) || fputc('\n', writer->manifest) == EOF)) goto done; ok = 1; done: archive_entry_free(entry); free(target); free(procpath); if (!ok) fprintf(stderr, "holypkg: cannot pack symlink\n"); return ok; } static int exact_members(int dir, const char *const *names, size_t count) { int copy = dup(dir); DIR *list; struct dirent *entry; unsigned seen = 0; size_t i; int ok = 1; if (copy < 0) return 0; list = fdopendir(copy); if (!list) { close(copy); return 0; } if (count >= sizeof seen * 8) { closedir(list); return 0; } errno = 0; while ((entry = readdir(list))) { if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; for (i = 0; i < count; ++i) if (!strcmp(entry->d_name, names[i])) break; if (i == count || (seen & (1u << i))) { ok = 0; break; } seen |= 1u << i; errno = 0; } if (!entry && errno) ok = 0; if (seen != (1u << count) - 1u) ok = 0; closedir(list); return ok; } static int compare_names(const void *a, const void *b) { return strcmp(*(const char *const *)a, *(const char *const *)b); } static int outside_tree(int root, const char *output) { const char *slash = strrchr(output, '/'); char *parent = slash ? strndup(output, slash == output ? 1 : (size_t)(slash - output)) : strdup("."); struct stat source, current, above; int dir = -1, ok = 0; unsigned depth; if (!parent || fstat(root, &source)) goto done; dir = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC); if (dir < 0) goto done; for (depth = 0; depth < 256; ++depth) { int next; if (fstat(dir, ¤t)) goto done; if (current.st_dev == source.st_dev && current.st_ino == source.st_ino) goto done; next = openat(dir, "..", O_RDONLY | O_DIRECTORY | O_CLOEXEC); if (next < 0) goto done; if (fstat(next, &above)) { close(next); goto done; } close(dir); dir = next; if (above.st_dev == current.st_dev && above.st_ino == current.st_ino) { ok = 1; goto done; } } done: if (dir >= 0) close(dir); free(parent); return ok; } static int sync_parent(const char *output) { const char *slash = strrchr(output, '/'); char *parent = slash ? strndup(output, slash == output ? 1 : (size_t)(slash - output)) : strdup("."); int dir, ok; if (!parent) return 0; dir = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC); free(parent); if (dir < 0) return 0; ok = !fsync(dir); close(dir); return ok; } static int walk_data(struct writer *writer, int parent, const char *relative, const char *archive_prefix, unsigned depth) { int dir = -1, scan = -1, ok = 0; DIR *list = NULL; struct dirent *item; char **names = NULL; size_t count = 0, capacity = 0, i; if (depth > 128) return 0; dir = openat(parent, relative, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (dir < 0 || flistxattr(dir, NULL, 0) != 0) goto done; scan = dup(dir); if (scan < 0 || !(list = fdopendir(scan))) goto done; scan = -1; errno = 0; while ((item = readdir(list))) { char **grown; if (!strcmp(item->d_name, ".") || !strcmp(item->d_name, "..")) continue; if (count == capacity) { size_t next = capacity ? capacity * 2 : 16; if (next < capacity || next > SIZE_MAX / sizeof *names) goto done; grown = realloc(names, next * sizeof *names); if (!grown) goto done; names = grown; capacity = next; } names[count] = strdup(item->d_name); if (!names[count]) goto done; ++count; errno = 0; } if (errno) goto done; if (count) qsort(names, count, sizeof *names, compare_names); for (i = 0; i < count; ++i) { char *path; struct stat st; size_t prefix = strlen(archive_prefix), component = strlen(names[i]); int directory; if (prefix > SIZE_MAX - component - 2) goto done; path = malloc(prefix + component + 2); if (!path) goto done; snprintf(path, prefix + component + 2, "%s/%s", archive_prefix, names[i]); if (fstatat(dir, names[i], &st, AT_SYMLINK_NOFOLLOW)) { free(path); goto done; } directory = S_ISDIR(st.st_mode); if (S_ISLNK(st.st_mode)) { int written = write_symlink(writer, dir, names[i], path); free(path); if (!written) goto done; continue; } if ((!directory && !S_ISREG(st.st_mode)) || !write_entry(writer, dir, names[i], path, directory) || (directory && !walk_data(writer, dir, names[i], path, depth + 1))) { free(path); goto done; } free(path); } ok = 1; done: for (i = 0; i < count; ++i) free(names[i]); free(names); if (list) closedir(list); if (scan >= 0) close(scan); if (dir >= 0) close(dir); return ok; } static int collect_files(struct writer *writer, int root) { size_t i, j, end; writer->collecting = 1; if (!walk_data(writer, root, "DATA", "DATA", 0)) return 0; writer->collecting = 0; if (writer->count) qsort(writer->files, writer->count, sizeof *writer->files, file_order); for (i = 0; i < writer->count; i = end) { end = i + 1; while (end < writer->count && writer->files[i].state.st_dev == writer->files[end].state.st_dev && writer->files[i].state.st_ino == writer->files[end].state.st_ino) ++end; for (j = i; j < end; ++j) { if (!unchanged(&writer->files[i].state, &writer->files[j].state)) return 0; writer->files[j].anchor = i; writer->files[j].members = end - i; } } return 1; } int holy_pack(const char *tree, const char *output) { static const char *const meta[] = { "meta", "files", "deps", "provides", "hooks", "origin", "transform" }; static const char *const roots[] = { "HOLY", "DATA" }; struct writer writer = {0}; char *temporary = NULL; int root = -1, holy = -1, data = -1, fd = -1, archive_fd = -1, ok = 0; size_t i, length = strlen(output); if (length > SIZE_MAX - 20) return 0; temporary = malloc(length + 20); if (!temporary) return 0; snprintf(temporary, length + 20, "%s.holy-tmp-XXXXXX", output); root = open(tree, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (root < 0 || !outside_tree(root, output)) goto done; holy = openat(root, "HOLY", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); data = openat(root, "DATA", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (holy < 0 || data < 0 || flistxattr(holy, NULL, 0) != 0 || flistxattr(data, NULL, 0) != 0 || !exact_members(root, roots, 2) || !exact_members(holy, meta, sizeof meta / sizeof *meta) || !collect_files(&writer, root)) goto done; fd = mkstemp(temporary); if (fd < 0) goto done; archive_fd = dup(fd); if (archive_fd < 0) goto done; writer.archive = archive_write_new(); if (!writer.archive || archive_write_add_filter_lz4(writer.archive) != ARCHIVE_OK || archive_write_set_format_pax_restricted(writer.archive) != ARCHIVE_OK || archive_write_set_options(writer.archive, "hdrcharset=UTF-8") != ARCHIVE_OK || archive_write_open_fd(writer.archive, archive_fd) != ARCHIVE_OK) goto done; if (!write_entry(&writer, root, "HOLY", "HOLY", 1)) goto done; for (i = 0; i < sizeof meta / sizeof *meta; ++i) { char path[32]; snprintf(path, sizeof path, "HOLY/%s", meta[i]); if (!write_entry(&writer, holy, meta[i], path, 0)) goto done; } if (!write_entry(&writer, root, "DATA", "DATA", 1) || !walk_data(&writer, root, "DATA", "DATA", 0) || writer.seen != writer.count) goto done; if (archive_write_close(writer.archive) != ARCHIVE_OK) goto done; archive_write_free(writer.archive); writer.archive = NULL; if (fcntl(archive_fd, F_GETFD) >= 0) close(archive_fd); archive_fd = -1; if (fsync(fd) || close(fd)) { fd = -1; goto done; } fd = -1; if (!holy_verify_with_output(temporary, 0) || !holy_scan_local_with_output(temporary, 0) || !holy_deps_local_with_output(temporary, 0) || !holy_provides_local(temporary, 0) || link(temporary, output)) goto done; if (!sync_parent(output)) { fprintf(stderr, "holypkg: output published but directory sync failed: %s\n", output); goto done; } printf("packed %s\n", output); ok = 1; done: if (!ok) fprintf(stderr, "holypkg: pack failed\n"); if (writer.archive) archive_write_free(writer.archive); free_files(&writer); if (archive_fd >= 0 && fcntl(archive_fd, F_GETFD) >= 0) close(archive_fd); if (fd >= 0) close(fd); if (temporary) { unlink(temporary); free(temporary); } if (data >= 0) close(data); if (holy >= 0) close(holy); if (root >= 0) close(root); return ok; } int holy_generate_files(const char *tree, const char *output) { struct writer writer = {0}; char *temporary = NULL; size_t length = strlen(output); int root = -1, fd = -1, ok = 0; if (length > SIZE_MAX - 20) return 0; temporary = malloc(length + 20); if (!temporary) return 0; snprintf(temporary, length + 20, "%s.holy-tmp-XXXXXX", output); root = open(tree, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (root < 0 || !outside_tree(root, output) || !collect_files(&writer, root)) goto done; fd = mkstemp(temporary); if (fd < 0) goto done; writer.manifest = fdopen(fd, "w"); if (!writer.manifest) goto done; fd = -1; if (!walk_data(&writer, root, "DATA", "DATA", 0) || writer.seen != writer.count || fflush(writer.manifest) || fsync(fileno(writer.manifest))) goto done; if (fclose(writer.manifest)) { writer.manifest = NULL; goto done; } writer.manifest = NULL; if (link(temporary, output)) goto done; if (!sync_parent(output)) { fprintf(stderr, "holypkg: manifest published but directory sync failed: %s\n", output); goto done; } printf("manifest %s\n", output); ok = 1; done: if (!ok) fprintf(stderr, "holypkg: manifest generation failed\n"); if (writer.manifest) fclose(writer.manifest); free_files(&writer); if (fd >= 0) close(fd); if (temporary) { unlink(temporary); free(temporary); } if (root >= 0) close(root); return ok; } int holy_pack_stream(int spool, const struct holy_stream_entry *entries, size_t count, int directory, const char *name) { struct archive *writer = NULL; struct archive_entry *entry = NULL; struct stat st; char temporary[43] = {0}, inspect[64], buffer[65536]; int fd = -1, copy = -1, ok = 0; size_t i; const char **paths = NULL; if (!*name || strchr(name, '/') || !strcmp(name, ".") || !strcmp(name, "..") || fstat(spool, &st) || !S_ISREG(st.st_mode) || st.st_size < 0) return 0; if (count > SIZE_MAX / sizeof *paths || !(paths = malloc(count * sizeof *paths))) return 0; for (i = 0; i < count; ++i) { const struct holy_stream_entry *e = &entries[i]; if (!holy_safe_archive_path(e->path) || e->size < 0 || e->offset < 0 || e->uid < 0 || e->gid < 0 || (e->mode & ~07777u) || ((e->directory || e->link || e->hardlink) && e->size) || (!e->directory && !e->link && !e->hardlink && (e->offset > st.st_size || e->size > st.st_size - e->offset))) goto done; paths[i] = e->path; } qsort(paths, count, sizeof *paths, compare_names); for (i = 1; i < count; ++i) if (!strcmp(paths[i-1], paths[i])) goto done; fd = holy_temporary_at(directory, temporary); if (fd < 0 || (copy = dup(fd)) < 0 || !(writer = archive_write_new())) goto done; if (archive_write_add_filter_lz4(writer) != ARCHIVE_OK || archive_write_set_format_pax_restricted(writer) != ARCHIVE_OK || archive_write_set_options(writer, "hdrcharset=UTF-8") != ARCHIVE_OK || archive_write_open_fd(writer, copy) != ARCHIVE_OK) goto done; for (i = 0; i < count; ++i) { const struct holy_stream_entry *e = &entries[i]; long long offset = 0; entry = archive_entry_new(); if (!entry) goto done; archive_entry_set_pathname(entry, e->path); archive_entry_set_filetype(entry, e->directory ? AE_IFDIR : e->link ? AE_IFLNK : AE_IFREG); archive_entry_set_perm(entry, e->mode); archive_entry_set_uid(entry, e->uid); archive_entry_set_gid(entry, e->gid); if (e->owner) archive_entry_set_uname(entry, e->owner); if (e->group) archive_entry_set_gname(entry, e->group); if (e->link) archive_entry_set_symlink(entry, e->link); if (e->hardlink) archive_entry_set_hardlink(entry, e->hardlink); archive_entry_set_size(entry, e->size); archive_entry_set_mtime(entry, 0, 0); if (archive_write_header(writer, entry) != ARCHIVE_OK) goto done; while (offset < e->size) { size_t wanted = e->size - offset < (long long)sizeof buffer ? (size_t)(e->size - offset) : sizeof buffer; ssize_t got = pread(spool, buffer, wanted, (off_t)(e->offset + offset)); if (got < 0 && errno == EINTR) continue; if (got <= 0 || archive_write_data(writer, buffer, (size_t)got) != got) goto done; offset += got; } archive_entry_free(entry); entry = NULL; } if (archive_write_close(writer) != ARCHIVE_OK) goto done; archive_write_free(writer); writer = NULL; if (fcntl(copy, F_GETFD) >= 0) close(copy); copy = -1; if (fsync(fd)) goto done; snprintf(inspect, sizeof inspect, "/proc/self/fd/%d", fd); if (!holy_verify_with_output(inspect, 0) || !holy_scan_local_with_output(inspect, 0) || !holy_deps_local_with_output(inspect, 0) || !holy_provides_local(inspect, 0) || linkat(directory, temporary, directory, name, 0) || fsync(directory)) goto done; ok = 1; done: free(paths); if (entry) archive_entry_free(entry); if (writer) archive_write_free(writer); if (copy >= 0 && fcntl(copy, F_GETFD) >= 0) close(copy); if (fd >= 0) close(fd); if (temporary[0]) unlinkat(directory, temporary, 0); return ok; }