old C version of package and build manager for sabotage linux, current version is written in shell/awk and can be found in KEEP/bin in the `sabotage` repo.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1073 lines
32 KiB

/*
Copyright (C) 2011-2015 rofl0r
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma RcB2 CFLAGS "-std=gnu99"
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <spawn.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/wait.h>
#include <time.h>
#include <errno.h>
#include <assert.h>
#include "../lib/include/stringptrlist.h"
#include "../lib/include/stringptr.h"
#include "../lib/include/strlib.h"
#include "../lib/include/logger.h"
#include "../lib/include/fileparser.h"
#include "../lib/include/iniparser.h"
#include "../lib/include/filelib.h"
#include "../lib/include/timelib.h"
#include "../lib/include/macros.h"
#include "../lib/include/hashlist.h"
#include "../lib/include/sha512.h"
#define VERSION "0.6.1"
#ifndef NUM_DL_THREADS
#define NUM_DL_THREADS 16
#endif
#ifndef NUM_BUILD_THREADS
#define NUM_BUILD_THREADS 1
#endif
#ifndef SLEEP_MS
#define SLEEP_MS 100
#endif
#define OPTIONS_SEPARATOR '.'
typedef enum {
PKGC_NONE = 0,
PKGC_INSTALL,
PKGC_REBUILD,
PKGC_PREFETCH,
PKGC_UPDATE,
} pkgcommands;
typedef enum {
DT_NONE = 0,
DT_BUILD = 1 << 0, /* deps required for building a package for a target */
DT_HOST= 1 << 1, /* deps required on the build host to build a package (for example perl if buildsys runs perl scripts) */
DT_RUN = 1 << 2, /* deps required to use full functionality of a package (for example if a package needs external programs to execute its binaries) */
DT_ALL = DT_BUILD | DT_HOST | DT_RUN,
} deptypes;
typedef struct {
stringptr* name;
stringptrlist* deps;
stringptrlist* mirrors;
stringptrlist* buildscript;
stringptrlist* vars;
stringptrlist* options;
} pkgdata;
typedef struct {
stringptr* filename;
stringptr* stdoutfn;
} scriptinfo;
typedef struct {
stringptr* name;
pid_t pid;
posix_spawn_file_actions_t fa;
scriptinfo scripts;
} pkg_exec;
typedef struct {
stringptr installroot;
stringptr pkgroot;
stringptr filecache;
stringptr arch;
stringptr logdir;
stringptr builddir;
stringptr keep;
stringptr butch_db;
} pkgconfig;
typedef struct {
unsigned avail;
unsigned max;
} procslots;
typedef enum {
JT_DOWNLOAD = 0,
JT_BUILD,
JT_MAX,
} jobtype;
struct installed_packages {
stringptrlist* names;
stringptrlist* hashes;
};
typedef struct {
pkgconfig cfg;
struct installed_packages installed_packages;
hashlist* package_list;
sblist* queue[JT_MAX];
stringptrlist* checked[JT_MAX];
stringptrlist* errors[JT_MAX];
stringptrlist* skippkgs;
procslots slots[JT_MAX];
int depflags;
} pkgstate;
static const char* queue_names[] = {
[JT_DOWNLOAD] = "download",
[JT_BUILD] = "build",
};
static const char* template_env_vars[] = {
[JT_DOWNLOAD] = "BUTCH_DOWNLOAD_TEMPLATE",
[JT_BUILD] = "BUTCH_BUILD_TEMPLATE"
};
#define PID_WAITING ((pid_t) -1)
#define PID_FINISHED ((pid_t) 0)
__attribute__((noreturn))
static void die(stringptr* message) {
log_puts(2, message);
exit(1);
}
__attribute__((noreturn))
static void die_errno(const char* msg) {
log_puterror(2, msg);
exit(1);
}
static void syntax(void) {
die(SPL(
"BUTCH v" VERSION "\n"
"syntax: butch command options\n\n"
"commands: install, rebuild, prefetch, update\n\n"
"pass an arbitrary number of package names as options\n\n"
"\tinstall: installs one or more packages when they're not yet installed\n"
"\t\t(list of installed packages is kept in /var/lib/butch.db unless\n"
"\t\t overridden via BUTCHDB env var.)\n"
"\trebuild: installs one or more packages even when they're already\n"
"\t\tinstalled\n"
"\tprefetch: only download the given package and all of its dependencies,\n"
"\t\tunless they're not already in $C\n"
"\tupdate: rebuild all packages that changed since last build\n"
"\n"
));
}
static ptrdiff_t in_skip_list(pkgstate *state, stringptr* pkg) {
stringptr* tmp;
if(!state->skippkgs) return -1;
sblist_iter_counter(state->skippkgs, i, tmp) {
if(EQ(tmp, pkg)) return i;
}
return -1;
}
/* entries in skiplist will be treated as if they failed to build
* this allows to skip over failing packages and their dependencies
* without the need to try to build them again. */
static void getconfig_skip(pkgstate* state) {
stringptr tmp;
state->skippkgs = 0;
stringptr_fromchar(getenv("BUTCH_SKIPLIST"), &tmp);
if(tmp.size) {
state->skippkgs = stringptr_splitc(&tmp, ':');
stringptrlist_dup_entries(state->skippkgs);
}
}
static int getconfig_deps(pkgstate* state) {
stringptr tmp;
stringptr_fromchar(getenv("DEPS"), &tmp);
if(!tmp.size) return DT_ALL;
stringptrlist *l = stringptr_splitc(&tmp, ':');
stringptr *item;
int res = 0;
sblist_iter(l, item) {
if(EQ(item, SPL("all"))) return DT_ALL;
if(EQ(item, SPL("host"))) res |= DT_HOST;
else if(EQ(item, SPL("run"))) res |= DT_RUN;
else if(EQ(item, SPL("build"))) res |= DT_BUILD;
}
return res;
}
static void getconfig(pkgstate* state) {
pkgconfig* c = &state->cfg;
stringptr_fromchar(getenv("A"), &c->arch);
stringptr_fromchar(getenv("R"), &c->installroot);
stringptr_fromchar(getenv("S"), &c->pkgroot);
stringptr_fromchar(getenv("B"), &c->builddir);
stringptr_fromchar(getenv("C"), &c->filecache);
stringptr_fromchar(getenv("K"), &c->keep);
stringptr_fromchar(getenv("LOGPATH"), &c->logdir);
stringptr_fromchar(getenv("BUTCHDB"), &c->butch_db);
if(!c->arch.size) {
die(SPL("need to set $A to your arch (i.e. x86_64, i386, arm, mips, ...)\n"));
}
if(!c->installroot.size) c->installroot = *(stringptr_copy(SPL("/")));
if(!c->pkgroot.size) c->pkgroot = *(stringptr_copy(SPL("/src")));
if(!c->builddir.size) c->builddir = *(stringptr_concat(&c->pkgroot, SPL("/build"), SPNIL));
if(!c->filecache.size) c->filecache = *(stringptr_copy(SPL("/src/tarballs")));
if(!c->keep.size) c->keep = *(stringptr_copy(SPL("/src/KEEP")));
if(!c->logdir.size) c->logdir = *(stringptr_copy(SPL("/src/logs")));
if(!c->butch_db.size) c->butch_db = *(stringptr_copy(SPL("/var/lib/butch.db")));
#define check_access(X, MODE) if(access(c->X.ptr, MODE) == -1) { \
log_put(2, VARISL("cannot access "), VARISL(#X), VNIL); \
log_perror(c->X.ptr); \
die(SPL("check your environment vars, if the directory exists and\nthat you have sufficient permissions (may need root)\n")); \
} /* "" */
check_access(logdir, W_OK);
check_access(installroot, W_OK);
check_access(pkgroot, R_OK);
check_access(filecache, W_OK);
check_access(keep, R_OK);
if(access(c->builddir.ptr, W_OK) == -1 && (errno != ENOENT || mkdir(c->builddir.ptr, 0770) == -1)) {
check_access(builddir, W_OK);
}
char buf[256], *p;
ulz_snprintf(buf, sizeof buf, "%s", c->butch_db.ptr);
if((p=strrchr(buf, '/'))) {
*p = 0;
if(access(buf, W_OK) == -1 && (errno != ENOENT || mkdir(buf, 0770) == -1)) {
die(stringptr_concat(SPL("directory for "), &c->butch_db, SPL(" could not be created or no write perm.\n"), SPNIL));
}
}
#undef check_access
int i;
for (i=0;i<JT_MAX;i++)
if(!getenv(template_env_vars[i]))
die(stringptr_format("required env var %s not set!\n", template_env_vars[i]));
getconfig_skip(state);
state->depflags = getconfig_deps(state);
}
/* outbuf must be at least 128+1 bytes */
static void sha512_to_str(const unsigned char hash[64],char outbuf[129]) {
size_t i;
for (i = 0; i<64; ++i) {
outbuf[2 * i] = "0123456789abcdef"[15 & (hash[i] >> 4)];
outbuf[2 * i + 1] = "0123456789abcdef"[15 & hash[i]];
}
outbuf[2 * i] = 0;
}
static int sha512_hash(const char* filename, char outbuf[129]) {
int fd;
sha512_ctx ctx;
ssize_t nread;
char buf[4*1024];
int success = 0;
fd = open(filename, O_RDONLY);
if(fd == -1) return 0;
sha512_init(&ctx);
while(1) {
nread = read(fd, buf, sizeof(buf));
if(nread < 0) goto err;
else if(nread == 0) break;
sha512_update(&ctx, (const uint8_t*) buf, nread);
}
success = 1;
unsigned char* hash = sha512_end(&ctx);
sha512_to_str(hash, outbuf);
err:
close(fd);
return success;
}
static void get_package_filename(pkgstate *state, stringptr* packagename, char* buf, size_t buflen) {
ulz_snprintf(buf, buflen, "%s/pkg/%s", state->cfg.pkgroot.ptr, packagename->ptr);
}
static int package_exists(pkgstate *state, stringptr* packagename) {
char buf[256];
get_package_filename(state, packagename, buf, sizeof(buf));
return access(buf, R_OK) == 0;
}
static int get_package_hash(pkgstate *state, stringptr* packagename, char* outbuf) {
char buf[256];
get_package_filename(state, packagename, buf, sizeof(buf));
return sha512_hash(buf, outbuf);
}
static void add_var(stringptrlist *list, stringptr *key, stringptr *value) {
stringptr *temp = stringptr_concat(key, SPL("="), value, SPNIL);
stringptrlist_add_strdup(list, temp);
stringptr_free(temp);
}
static void package_get_deps(pkgstate *state, stringptrlist* ini, pkgdata* out) {
ini_section sec = iniparser_get_section(ini, SPL("deps"));
if(!out->deps) out->deps = stringptrlist_new(sec.linecount);
static const struct { const stringptr secname; deptypes dt; } depmap[] = {
{ .secname = SPINITIALIZER("deps"), .dt = DT_BUILD },
{ .secname = SPINITIALIZER("deps.host"), .dt = DT_HOST },
{ .secname = SPINITIALIZER("deps.run"), .dt = DT_RUN },
};
for(size_t i = 0; i < ARRAY_SIZE(depmap); i++) {
if(state->depflags & depmap[i].dt) {
sec = iniparser_get_section(ini, &depmap[i].secname);
for(size_t start = sec.startline; start < sec.startline + sec.linecount; start++) {
stringptr *tmp = stringptrlist_get(ini, start);
if(tmp->size && in_skip_list(state, tmp) == -1) stringptrlist_add_strdup(out->deps, tmp);
}
}
}
}
static void package_get_vars(pkgstate* state, stringptrlist* ini, pkgdata* out) {
(void) state;
ini_section sec = iniparser_get_section(ini, SPL("vars"));
if(!out->vars) out->vars = stringptrlist_new(sec.linecount ? sec.linecount : 1);
for(size_t start = sec.startline; start < sec.startline + sec.linecount; start++) {
stringptr *tmp = stringptrlist_get(ini, start);
stringptrlist_add_strdup(out->vars, tmp);
}
}
static void ini_from_packagename(pkgstate *state,
stringptr* packagename, stringptr* option,
stringptr** out_fc, stringptrlist **out_ini) {
stringptr* pname = packagename;
*out_fc = 0, *out_ini = 0;
if(option) {
static const char sepbuf[2] = { OPTIONS_SEPARATOR, 0};
static const stringptr sep = {.ptr = (char*)sepbuf, .size = 1};
pname = stringptr_concat(packagename, &sep, option, SPNIL);
if(!pname) return;
}
char buf[256];
get_package_filename(state, pname, buf, sizeof(buf));
if(pname != packagename) stringptr_free(pname);
*out_fc = stringptr_fromfile(buf);
if(!*out_fc) return;
*out_ini = stringptr_linesplit(*out_fc);
}
// contract: out is already zeroed and contains only name and options
static void get_package_contents(pkgstate *state, stringptr* packagename, pkgdata* out) {
stringptr* fc, val;
stringptrlist* ini;
ini_from_packagename(state, packagename, 0, &fc, &ini);
if(!fc) goto err;
size_t start = 0;
ini_section sec;
stringptr* tmp;
sec = iniparser_get_section(ini, SPL("mirrors"));
out->mirrors = stringptrlist_new(sec.linecount);
for(start = sec.startline; start < sec.startline + sec.linecount; start++) {
tmp = stringptrlist_get(ini, start);
if(tmp->size) stringptrlist_add_strdup(out->mirrors, tmp);
}
package_get_deps(state, ini, out);
package_get_vars(state, ini, out);
sec = iniparser_get_section(ini, SPL("main"));
iniparser_getvalue(ini, &sec, SPL("tardir"), &val);
if(val.size) add_var(out->vars, SPL("tardir"), &val);
iniparser_getvalue(ini, &sec, SPL("sha512"), &val);
if(val.size) add_var(out->vars, SPL("sha512"), &val);
iniparser_getvalue(ini, &sec, SPL("filesize"), &val);
if(val.size) add_var(out->vars, SPL("filesize"), &val);
sec = iniparser_get_section(ini, SPL("build")); // the build section has always to come last
if(sec.startline || sec.linecount) {
start = sec.startline;
sec = iniparser_file_as_section(ini); // iniparser may disinterpret lines starting with [
// so be sure to use the entire rest of the file
sec.startline = start;
sec.linecount -= start;
out->buildscript = stringptrlist_new(sec.linecount);
for(start = sec.startline; start < sec.startline + sec.linecount; start++) {
tmp = stringptrlist_get(ini, start);
stringptrlist_add_strdup(out->buildscript, tmp);
}
} else
out->buildscript = stringptrlist_new(1);
stringptrlist_free(ini);
stringptr_free(fc);
if(out->options) {
stringptr *o;
sblist_iter(out->options, o) {
ini_from_packagename(state, packagename, o, &fc, &ini);
tmp = stringptr_concat(SPL("option_"), o, SPNIL);
if(tmp) {
add_var(out->vars, tmp, SPL("1"));
stringptr_free(tmp);
}
if(ini) {
package_get_deps(state, ini, out);
package_get_vars(state, ini, out);
stringptrlist_free(ini);
}
if(fc) stringptr_free(fc);
}
}
return;
err:
log_perror(packagename->ptr);
die(SPL("package not existing\n"));
}
static void write_installed_dat(pkgstate* state);
static void get_installed_packages(pkgstate* state) {
fileparser f;
char buf[256];
stringptr line;
int oldformat = 0;
if(fileparser_open(&f, state->cfg.butch_db.ptr)) goto err;
while(!fileparser_readline(&f) && !fileparser_getline(&f, &line) && line.size) {
char* p = line.ptr;
while(*p && *p != ' ') p++;
*p = 0;
size_t l = (size_t) p - (size_t) line.ptr;
stringptr *temp = SPMAKE(line.ptr, l);
stringptrlist_add_strdup(state->installed_packages.names, temp);
if(l == line.size) {
/* old butch.db format containing only package names */
oldformat = 1;
get_package_hash(state, temp, buf);
temp = SPMAKE(buf, 128);
} else {
p++, l++;
temp = SPMAKE(p, line.size - l);
}
stringptrlist_add_strdup(state->installed_packages.hashes, temp);
}
fileparser_close(&f);
if(oldformat) write_installed_dat(state);
return;
err:
if(errno != ENOENT) log_perror("failed to open butch.db!");
}
static int is_installed(pkgstate* state, stringptr* packagename) {
return stringptrlist_contains(state->installed_packages.names, packagename);
}
static void free_package_data(pkgdata* data) {
stringptrlist_freeall(data->buildscript);
stringptrlist_freeall(data->deps);
stringptrlist_freeall(data->mirrors);
stringptrlist_freeall(data->vars);
if(data->options) stringptrlist_free(data->options);
stringptr_free(data->name);
}
static int is_in_queue(stringptr* packagename, sblist* queue) {
pkg_exec* item;
sblist_iter(queue, item) {
if(EQ(item->name, packagename))
return 1;
}
return 0;
}
static void add_queue(stringptr* packagename, sblist* queue) {
pkg_exec execdata = {.pid = PID_WAITING, .name = stringptr_copy(packagename)};
sblist_add(queue, &execdata);
}
static pkgdata* packagelist_get(hashlist* list, stringptr* name, uint32_t hash) {
sblist* bucket = hashlist_get(list, hash);
pkgdata* result;
if(bucket) {
sblist_iter(bucket, result) {
if(EQ(name, result->name))
return result;
}
}
return 0;
}
static pkgdata* packagelist_add(hashlist* list, stringptr* name, uint32_t hash) {
pkgdata pkg_empty = {.name = stringptr_copy(name)};
hashlist_add(list, hash, &pkg_empty);
return packagelist_get(list, name, hash);
}
/* returns a list of options *and* modifies packagename to contain only
the packagename part before the first OPTIONS_SEPARATOR */
stringptrlist* get_package_options(stringptr* packagename) {
char *q, *p = strchr(packagename->ptr, OPTIONS_SEPARATOR);
if(!p) return 0;
stringptrlist *o = stringptrlist_new(4);
if(!o) return 0;
packagename->size = p - packagename->ptr;
do {
*p = 0;
p++;
stringptr n = {.ptr = p};
q = strchr(p, OPTIONS_SEPARATOR);
n.size = q ? q - p : strlen(p);
p = q;
stringptrlist_add(o, n.ptr, n.size);
} while(q);
return o;
}
static void queue_package(pkgstate* state, stringptr* packagename, jobtype jt, int force) {
static int depth = 0;
depth++;
if(depth > 100) {
ulz_fprintf(2, "WARNING: recursion level above 100!\n");
goto end;
}
if(!packagename->size) goto end;
stringptrlist* options = get_package_options(packagename);
if(in_skip_list(state, packagename) >= 0) goto end;
sblist* queue = state->queue[jt];
stringptrlist* checklist = state->checked[jt];
// check if we already processed this entry.
if(stringptrlist_contains(checklist, packagename)) {
goto end;
}
stringptrlist_add_strdup(checklist, packagename);
if(is_in_queue(packagename, queue)) goto end;
uint32_t hash = stringptr_hash(packagename);
pkgdata* pkg = packagelist_get(state->package_list, packagename, hash);
unsigned i;
if(!pkg) {
pkg = packagelist_add(state->package_list, packagename, hash);
pkg->options = options;
options = 0;
get_package_contents(state, packagename, pkg);
}
for(i = 0; i < stringptrlist_getsize(pkg->deps); i++) {
queue_package(state, stringptrlist_get(pkg->deps, i), jt, 0); // omg recursion
pkg = packagelist_get(state->package_list, packagename, hash);
}
if(!force && is_installed(state, packagename)) {
ulz_fprintf(1, "package %s is already installed, skipping %s\n", packagename->ptr, queue_names[jt]);
goto end;
}
if(
// if sizeof mirrors is 0, it is a meta package
(jt == JT_DOWNLOAD && stringptrlist_getsize(pkg->mirrors))
|| (jt == JT_BUILD)
) {
add_queue(packagename, queue);
}
end:
if(options) stringptrlist_free(options);
depth--;
}
static stringptr* make_config(pkgconfig* cfg) {
#define EXPORT(K, V) SPL("export " K "="), V, SPL("\n")
stringptr* result = stringptr_concat(
EXPORT("A", &cfg->arch),
EXPORT("R", &cfg->installroot),
EXPORT("S", &cfg->pkgroot),
EXPORT("C", &cfg->filecache),
EXPORT("K", &cfg->keep),
EXPORT("B", &cfg->builddir),
SPNIL);
return result;
#undef EXPORT
}
static stringptr *get_mirror_urls(pkgdata* data) {
size_t i = 0;
stringptr *new = stringptr_new(0);
for(;i<stringptrlist_getsize(data->mirrors);i++)
new = stringptr_concat(new, i ? SPL(" ") : SPL(""),
stringptrlist_get(data->mirrors, i), SPNIL);
return new;
}
static int create_script(jobtype ptype, pkgstate* state, pkg_exec* item, pkgdata* data) {
stringptr *temp, *temp2, *config, *vars;
static const char* prefixes[] = { [JT_DOWNLOAD] = "dl", [JT_BUILD] = "build", };
const char *prefix = prefixes[ptype];
char *custom_template = getenv(template_env_vars[ptype]);
item->scripts.filename = stringptr_format("%s/%s_%s.sh", state->cfg.builddir.ptr, prefix, item->name->ptr);
item->scripts.stdoutfn = stringptr_format("%s/%s_%s.log", state->cfg.logdir.ptr, prefix, item->name->ptr);
temp = make_config(&state->cfg);
vars = stringptrlist_tostring(data->vars);
config = stringptr_concat(temp, vars, SPNIL);
stringptr_free(temp); stringptr_free(vars);
if(ptype == JT_BUILD && !stringptrlist_getsize(data->buildscript)) {
/* execute empty script when pkg has no build section */
temp = stringptr_copy(SPL("#!/bin/sh\ntrue\n"));
goto write_it;
}
stringptr* buildscr = (ptype == JT_BUILD ? stringptrlist_tostring(data->buildscript) : SPL(""));
temp = stringptr_fromfile(custom_template);
if(!temp) die(SPL("error reading custom_template, using default one\n"));
temp2 = stringptr_replace(temp, SPL("%BUTCH_CONFIG"), config);
stringptr_free(temp); temp = temp2;
temp2 = stringptr_replace(temp, SPL("%BUTCH_PACKAGE_NAME"), item->name);
stringptr_free(temp); temp = temp2;
temp2 = stringptr_replace(temp, SPL("%BUTCH_BUILDSCRIPT"), buildscr);
stringptr_free(temp); temp = temp2;
temp2 = stringptr_replace(temp, SPL("%BUTCH_IS_REBUILD"), is_installed(state, item->name) ? SPL("true") : SPL("false"));
stringptr_free(temp); temp = temp2;
stringptr *temp3 = get_mirror_urls(data);
temp2 = stringptr_replace(temp, SPL("%BUTCH_MIRROR_URLS"), temp3);
stringptr_free(temp3);
stringptr_free(temp); temp = temp2;
if(ptype == JT_BUILD) stringptr_free(buildscr);
write_it:
stringptr_tofile(item->scripts.filename->ptr, temp);
if(chmod(item->scripts.filename->ptr, 0775) == -1) die(SPL("error setting permission"));
stringptr_free(config);
stringptr_free(temp);
return 1;
}
extern char** environ;
static void launch_thread(jobtype ptype, pkgstate* state, pkg_exec* item, pkgdata* data) {
static const char* lt_msgs[] = { [JT_DOWNLOAD] = " downloading ", [JT_BUILD] = " building ", };
char* arr[2];
create_script(ptype, state, item, data);
log_timestamp(1);
log_put(1, VARICC(lt_msgs[ptype]), VARIS(item->name), VARISL(" ("), VARIS(item->scripts.filename), VARISL(") -> "), VARIS(item->scripts.stdoutfn), VNIL);
arr[0] = item->scripts.filename->ptr;
arr[1] = 0;
posix_spawn_file_actions_init(&item->fa);
posix_spawn_file_actions_addclose(&item->fa, 0);
posix_spawn_file_actions_addclose(&item->fa, 1);
posix_spawn_file_actions_addclose(&item->fa, 2);
posix_spawn_file_actions_addopen(&item->fa, 0, "/dev/null", O_RDONLY, 0);
posix_spawn_file_actions_addopen(&item->fa, 1, item->scripts.stdoutfn->ptr, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
posix_spawn_file_actions_adddup2(&item->fa, 1, 2);
int ret = posix_spawnp(&item->pid, arr[0], &item->fa, 0, arr, environ);
if(ret == -1) {
log_perror("posix_spawn");
die(SPL(""));
}
}
// checks if all dependencies are installed
// then checks if the tarball is downloaded
// then checks if its either a metapackage or doesnt require a tarball.
static int has_all_deps(pkgstate* state, pkgdata* item) {
size_t i;
pkg_exec* dlitem;
for(i = 0; i < stringptrlist_getsize(item->deps); i++) {
stringptr *s = stringptrlist_get(item->deps, i);
if(in_skip_list(state, s) == -1 && !is_installed(state, s)) return 0;
}
if(!stringptrlist_getsize(item->mirrors)) return 1;
sblist_iter(state->queue[JT_DOWNLOAD], dlitem) {
if(EQ(dlitem->name, item->name)) {
if(dlitem->pid == PID_FINISHED) { //download finished?
stringptr *s;
sblist_iter(state->errors[JT_DOWNLOAD], s) {
if(EQ(dlitem->name, s)) return 0;
}
return 1;
} else return 0;
}
}
return 0;
}
/* returns 1 if there are no unfinished (i.e. waiting or running)
* processes in the queue, otherwise 0 */
static int queue_empty(sblist* queue) {
pkg_exec* item;
sblist_iter(queue, item) {
if(item->pid != PID_FINISHED)
return 0;
}
return 1;
}
static void fill_slots(jobtype ptype, pkgstate* state) {
size_t i;
pkg_exec* item;
pkgdata* pkg;
unsigned* slots_avail = &state->slots[ptype].avail;
sblist* queue = state->queue[ptype];
for(i = 0; *slots_avail && i < sblist_getsize(queue); i++) {
item = sblist_get(queue, i);
if(item->pid == PID_WAITING) {
if(in_skip_list(state, item->name) >= 0) {
item->pid = PID_FINISHED;
continue;
}
pkg = packagelist_get(state->package_list, item->name, stringptr_hash(item->name));
if(ptype == JT_DOWNLOAD || has_all_deps(state, pkg)) {
launch_thread(ptype, state, item, pkg);
(*slots_avail)--;
}
}
}
}
static void prepare_slots(pkgstate* state) {
char *p;
p = getenv("BUTCH_DL_THREADS");
state->slots[JT_DOWNLOAD].max = p ? atoi(p) : NUM_DL_THREADS;
p = getenv("BUTCH_BUILD_THREADS");
state->slots[JT_BUILD].max = p ? atoi(p) : NUM_BUILD_THREADS;
state->slots[JT_DOWNLOAD].avail = state->slots[JT_DOWNLOAD].max;
state->slots[JT_BUILD].avail = state->slots[JT_BUILD].max;
fill_slots(JT_DOWNLOAD, state);
fill_slots(JT_BUILD, state);
}
static void print_queue(pkgstate* state, jobtype jt) {
sblist* queue = state->queue[jt];
const char *queuename = queue_names[jt];
pkg_exec* listitem;
log_put(1, VARISL("*** "), VARICC(queuename), VARISL("queue ***"), VNIL);
sblist_iter(queue, listitem) {
log_puts(1, listitem->name);
log_putln(1);
}
}
static void print_info(pkgstate* state) {
print_queue(state, JT_DOWNLOAD);
print_queue(state, JT_BUILD);
}
static void write_installed_dat(pkgstate* state) {
char buf[256];
char bak[256];
ulz_snprintf(buf, sizeof(buf), "%s", state->cfg.butch_db.ptr);
ulz_snprintf(bak, sizeof(bak), "%s.bak", state->cfg.butch_db.ptr);
/* block SIGINT */
struct sigaction old, nu;
int unblocksig = 0;
if(!sigaction(SIGINT, 0, &old)) {
unblocksig = 1;
nu = old;
nu.sa_handler = SIG_IGN;
nu.sa_flags &= ~SA_SIGINFO;
sigaction(SIGINT, &nu, 0);
}
int renamed = 1;
if(rename(buf, bak) == -1) {
renamed = 0;
if(errno != ENOENT) log_puterror(2, "trying to rename butch.db failed");
}
int fd = open(buf, O_CREAT | O_TRUNC | O_RDWR, 0664);
if(fd == -1) goto err;
size_t i;
assert(sblist_getsize(state->installed_packages.names) == sblist_getsize(state->installed_packages.hashes));
for(i = 0; i < sblist_getsize(state->installed_packages.names); i++) {
stringptr* s = stringptrlist_get(state->installed_packages.names, i);
if(write(fd, s->ptr, s->size) != s->size) goto err;
if(write(fd, " ", 1) != 1) goto err;
s = stringptrlist_get(state->installed_packages.hashes, i);
if(write(fd, s->ptr, s->size) != s->size) goto err;
if(write(fd, "\n", 1) != 1) goto err;
}
close(fd);
if(renamed) unlink(bak);
if(unblocksig) sigaction(SIGINT, &old, 0);
return;
err:
if(renamed) rename(bak, buf);
die(SPL("error writing to butch.db"));
}
static void mark_finished(pkgstate* state, stringptr* name) {
char hash[256];
if(!get_package_hash(state, name, hash)) log_puterror(2, "failed to get pkg hash");
ssize_t idx = stringptrlist_find(state->installed_packages.names, name);
if(idx == -1) {
stringptrlist_add_strdup(state->installed_packages.names, name);
stringptrlist_add_strdup(state->installed_packages.hashes, SPMAKE(hash, 128));
} else { /* update hash */
stringptr* e = stringptrlist_get(state->installed_packages.hashes, idx);
free(e->ptr);
char* e2 = stringptr_strdup(SPMAKE(hash, 128));
stringptrlist_set(state->installed_packages.hashes, idx, e2, 128);
}
write_installed_dat(state);
}
static void prepare_update(pkgstate* state, stringptrlist* packages2install) {
char hash[256];
size_t i;
for(i = 0; i < sblist_getsize(state->installed_packages.names);) {
stringptr* name = stringptrlist_get(state->installed_packages.names, i);
if(!package_exists(state, name)) goto next;
if(!get_package_hash(state, name, hash)) {
log_puterror(2, "failed to get pkg hash");
goto next;
}
stringptr* h = stringptrlist_get(state->installed_packages.hashes, i);
stringptr* h2 = SPMAKE(hash, 128);
if(!EQ(h, h2)) {
stringptrlist_add(packages2install, name->ptr, name->size);
free(h->ptr);
sblist_delete(state->installed_packages.names, i);
sblist_delete(state->installed_packages.hashes, i);
} else {
next:
i++;
}
}
}
static void warn_errors(pkgstate* state) {
size_t i;
stringptr* candidate;
for(i = 0; i < stringptrlist_getsize(state->errors[JT_BUILD]); i++) {
candidate = stringptrlist_get(state->errors[JT_BUILD], i);
log_put(2, VARISL("WARNING: "), VARIS(candidate), VARISL(" failed to build! wait for other jobs to finish."), VNIL);
}
}
static void check_finished_processes(pkgstate* state, jobtype jt, int* had_event) {
pkg_exec* listitem;
sblist *queue = state->queue[jt];
sblist_iter(queue, listitem) {
int exitstatus, ret;
// check for a running process.
if(listitem->pid == PID_FINISHED || listitem->pid == PID_WAITING) continue;
ret = waitpid(listitem->pid, &exitstatus, WNOHANG);
// still busy
if(ret == 0) continue;
*had_event = 1;
state->slots[jt].avail++;
posix_spawn_file_actions_destroy(&listitem->fa);
if(ret == -1) {
log_perror("waitpid");
listitem->pid = PID_WAITING;
continue;
}
if(exitstatus == 0) {
// process exited gracefully
if(jt == JT_DOWNLOAD) {
goto finished;
} else {
mark_finished(state, listitem->name);
goto finished;
}
} else {
if(jt == JT_DOWNLOAD)
log_put(2, VARISL("got error "), VARII(WEXITSTATUS(exitstatus)), VARISL(" from download script of "), VARIS(listitem->name), VNIL);
stringptrlist_add_strdup(state->errors[jt], listitem->name);
finished:
listitem->pid = PID_FINISHED;
}
}
}
static void check_processes_and_fill_slots(pkgstate* state, jobtype jt, int* had_event) {
check_finished_processes(state, jt, had_event);
if(state->slots[jt].avail) fill_slots(jt, state);
}
static int process_queue(pkgstate* state) {
int had_event = 0;
check_processes_and_fill_slots(state, JT_DOWNLOAD, &had_event);
check_processes_and_fill_slots(state, JT_BUILD, &had_event);
if(had_event) warn_errors(state);
int done = (state->slots[JT_DOWNLOAD].avail == state->slots[JT_DOWNLOAD].max &&
state->slots[JT_BUILD].avail == state->slots[JT_BUILD].max);
return !done;
}
static void freequeue(sblist* queue) {
pkg_exec* pe;
sblist_iter(queue, pe) {
stringptr_free(pe->name);
stringptr_free(pe->scripts.filename);
stringptr_free(pe->scripts.stdoutfn);
}
sblist_free(queue);
}
int main(int argc, char** argv) {
pkgstate state;
pkgcommands mode = PKGC_NONE;
int i;
const char* opt_strings[] = {
[PKGC_INSTALL] = "install",
[PKGC_REBUILD] = "rebuild",
[PKGC_PREFETCH] = "prefetch",
[PKGC_UPDATE] = "update",
};
if(argc < 2) syntax();
for(i = PKGC_NONE + 1; (unsigned) i < ARRAY_SIZE(opt_strings); i++)
if(!strcmp(argv[1], opt_strings[i]))
mode = (pkgcommands) i;
if(mode == PKGC_NONE || (mode != PKGC_UPDATE && argc < 3) || (mode == PKGC_UPDATE && argc > 2))
syntax();
/* if /dev/null is missing posix_spawn would silently fail when executing child processes */
if(access("/dev/null", R_OK) == -1) {
perror("error accessing /dev/null");
die(SPL(""));
}
srand(time(0));
getconfig(&state);
state.installed_packages.names = stringptrlist_new(64);
state.installed_packages.hashes = stringptrlist_new(64);
get_installed_packages(&state);
state.package_list = hashlist_new(64, sizeof(pkgdata));
state.queue[JT_DOWNLOAD] = sblist_new(sizeof(pkg_exec), 64);
state.queue[JT_BUILD] = sblist_new(sizeof(pkg_exec), 64);
state.errors[JT_DOWNLOAD] = stringptrlist_new(4);
state.errors[JT_BUILD] = stringptrlist_new(4);
state.checked[JT_DOWNLOAD] = stringptrlist_new(64);
state.checked[JT_BUILD] = stringptrlist_new(64);
stringptrlist* packages2install = stringptrlist_new(16);
if(mode == PKGC_UPDATE) {
prepare_update(&state, packages2install);
mode = PKGC_INSTALL;
} else for(i = 2; i < argc; i++) {
stringptr curr;
// allow something like pkg/packagename to be passed
char* pkg_name = strrchr(argv[i], '/');
if(!pkg_name) pkg_name = argv[i];
else pkg_name++;
stringptr_fromchar(pkg_name, &curr);
stringptrlist_add_strdup(packages2install, &curr);
}
stringptr *curr_pkg;
sblist_iter(packages2install, curr_pkg) {
const int force[] = {
[PKGC_REBUILD] = 1,
[PKGC_INSTALL] = 0,
[PKGC_PREFETCH] = 1 };
queue_package(&state, curr_pkg, JT_DOWNLOAD, force[mode]);
if(mode != PKGC_PREFETCH)
queue_package(&state, curr_pkg, JT_BUILD, force[mode]);
}
print_info(&state);
prepare_slots(&state);
while(process_queue(&state)) msleep(SLEEP_MS);
stringptrlist_freeall(packages2install);
int failed = stringptrlist_getsize(state.errors[JT_BUILD]) != 0;
if(state.skippkgs) goto skipfailure_check;
if(!failed && (!(queue_empty(state.queue[JT_DOWNLOAD])) || !(queue_empty(state.queue[JT_BUILD])))) {
ulz_fprintf(2, "WARNING: circular reference or download error!\n");
failed = 1;
}
skipfailure_check:
// clean up ...
stringptrlist_freeall(state.errors[JT_DOWNLOAD]);
stringptrlist_freeall(state.errors[JT_BUILD]);
stringptrlist_freeall(state.checked[JT_DOWNLOAD]);
stringptrlist_freeall(state.checked[JT_BUILD]);
stringptrlist_freeall(state.installed_packages.names);
stringptrlist_freeall(state.installed_packages.hashes);
if(state.skippkgs)
stringptrlist_freeall(state.skippkgs);
hashlist_iterator hit;
hashlist_iterator_init(&hit);
pkgdata* data;
while((data = hashlist_next(state.package_list, &hit))) {
free_package_data(data);
}
hashlist_free(state.package_list);
freequeue(state.queue[JT_DOWNLOAD]);
freequeue(state.queue[JT_BUILD]);
log_timestamp(1);
log_putspace(1);
log_puts(1, SPL("done."));
log_putln(1);
return failed;
}