The real reason I wrote this is to have an automated way to set up the pkg_add mirrors especially for folks that don't care to set them up manually (Afterall, that's what computers are for!). Before I wrote this, I had a PKG_PATH mirror go down and I didn't know what was going on. At least this could get some failover that would work for everyone running the release or older at least. I put in a minor edit that kills ftp and sed if the buffer gets too full, as well as exiting.
> > > >> > > I have a 500 line program I wrote that reads openbsd.org.ftp.html and >> >> Here's a simple alternative that will often be good enough. >> >> ftp -o- -V http://www.openbsd.org/cgi-bin/ftplist.cgi | >> sed -e 's, .*,/%m/,' -e 's,^,pkgpath = ,' -e q >> >> The C program is too trusting with its fixed-size buffers and unchecked >> mallocs etc, it's not something to run as root as-is. >> > >
/* * Copyright (c) 2016 Luke N. Small * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Special thanks to Dan Mclaughlin for the ftp to sed idea * * ftp -o - http://www.openbsd.org/ftp.html | \ * sed -n \ * -e 's: <strong>\([^<]*\)<.*:\1 :p' \ * -e 's:^\( [hfr].*\):\1:p' */ #define EVENT_NOPOLL #define EVENT_NOSELECT #include <stdio.h> #include <unistd.h> #include <stdlib.h> #include <err.h> #include <sys/types.h> #include <sys/event.h> #include <signal.h> #include <string.h> struct mirror_st { char * countryTitle; char * mirror; char * installPath; double diff; struct mirror_st * next; }; int ftp_cmp (const void * a, const void * b) { struct mirror_st **one = (struct mirror_st **)a; struct mirror_st **two = (struct mirror_st **)b; if( (*one)->diff < (*two)->diff ) return -1; if( (*one)->diff > (*two)->diff ) return 1; return 0; } int country_cmp (const void * a, const void * b) { struct mirror_st **one = (struct mirror_st **)a; struct mirror_st **two = (struct mirror_st **)b; // list the USA mirrors first, it will subsort correctly int8_t temp = !strncmp("USA", (*one)->countryTitle, 3); if(temp != !strncmp("USA", (*two)->countryTitle, 3)) { if(temp) return -1; return 1; } return strcmp( (*one)->countryTitle, (*two)->countryTitle ) ; } double getTimeDiff(struct timeval a, struct timeval b) { long sec; long usec; sec = b.tv_sec - a.tv_sec; usec = b.tv_usec - a.tv_usec; if (usec < 0) { --sec; usec += 1000000; } return sec + ((double)usec / 1000000.0); } int main() { pid_t ftpPid, sedPid; int ftpToSed[2]; int sedToParent[2]; int unameToParent[2]; char unameR[5], unameM[20]; int i = 0, c; register int position, num; FILE *input; pipe(unameToParent); // "uname -rm" returns version and architecture like: "5.8 amd64" to standard out if(fork() == (pid_t) 0) { /* uname child */ close(unameToParent[0]); dup2(unameToParent[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/uname","/usr/bin/uname", "-rm", NULL); err(1, "execl() failed\n"); } close(unameToParent[1]); input = fdopen (unameToParent[0], "r"); num = 0; position = -1; while ((c = getc(input)) != EOF) { if(num == 0) { if(position >= 5) err(1, "unameR[] got too long!\n"); if(c != ' ') unameR[++position] = c; else { unameR[position + 1] = '\0'; num = 1; position = -1; } } else { if(position >= 20) err(1, "unameM[] got too long!\n"); if(c != '\n') unameM[++position] = c; else unameM[position + 1] = '\0'; } } fclose (input); close(unameToParent[0]); pipe(ftpToSed); /*make pipes*/ struct kevent ke[2]; int kq = kqueue(); if (kq == -1) err(1, "kq!"); int kqProc = kqueue(); if (kqProc == -1) err(1, "kqProc!"); ftpPid = fork(); if(ftpPid == (pid_t) 0) { /*ftp child*/ close(ftpToSed[0]); dup2(ftpToSed[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/ftp","ftp", "-Vo", "-", "http://www.openbsd.org/ftp.html";, NULL); err(1, "execl() failed\n"); } EV_SET(ke, ftpPid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, &ftpPid); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); close(ftpToSed[1]); pipe(sedToParent); sedPid = fork(); if(sedPid == (pid_t) 0) { /* sed child */ close(sedToParent[0]); dup2(ftpToSed[0], STDIN_FILENO); /*attaching to pipe(s)*/ dup2(sedToParent[1], STDOUT_FILENO); execl("/usr/bin/sed","sed","-n","-e", "s:\t<strong>\\([^<]*\\)<.*:\\1 :p", "-e", "s:^\\(\t[hfr].*\\):\\1:p", NULL); kill(ftpPid, SIGKILL); err(1, "execl() failed\n"); } EV_SET(ke, sedPid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, &sedPid); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); close(ftpToSed[0]); /*close pipe(s) child attached to*/ close(sedToParent[1]); EV_SET(ke, sedToParent[0], EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, NULL); if (kevent(kq, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); struct timespec timeout; timeout.tv_sec = 7; timeout.tv_nsec = 0; i = kevent(kq, NULL, 0, ke, 1, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { close(sedToParent[0]); printf("timed out.\n"); kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); return -1; } input = fdopen (sedToParent[0], "r"); char line[200]; num = 0; position = 0; struct mirror_st *start, *end, *mTemp1, *mTemp2, *mTemp3; start = end = malloc(sizeof(struct mirror_st)); if(start == NULL) err(1, "malloc failed.\n"); start->next = NULL; while((c = getc (input)) != EOF) { if(position >= 200) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); err(1, "line[] got too long!\n"); } if(num == 0) { if( c != '\n' ) line[position++] = c; else { // there is a space before the newline that is eliminated line[position-1] = '\0'; end->countryTitle = malloc(position); if(end->countryTitle == NULL) err(1, "malloc failed.\n"); strlcpy(end->countryTitle, line, position); position = 0; num = 1; } } else { if(position == 0) { if( (c != 'h') && (c != 'f') && (c != 'r') ) continue; else if(c == 'r') break; else if(c == 'f') { // changes ftp listings to http. ftp.html says they can be either line[position++] = 'h'; line[position++] = 't'; continue; } } if( c != '\n' ) line[position++] = c; else { // there is a "</a>" before the newline that is overwritten line[position -= 4] = '\0'; ++position; position += num = strlen(unameR) + 10 + strlen(unameM) + 7; end->mirror = malloc(position); if(end->mirror == NULL) err(1, "malloc failed.\n"); c = strlcpy(end->mirror, line, position); c += strlcpy(end->mirror + c, unameR, position - c); c += strlcpy(end->mirror + c, "/packages/", position - c); c += strlcpy(end->mirror + c, unameM, position - c); strlcpy(end->mirror + c, "/SHA256", position - c); position += 15 - num; end->installPath = malloc(position); if(end->installPath == NULL) err(1, "malloc failed.\n"); c = strlcpy(end->installPath, line, position); strlcpy(end->installPath + c, "%c/packages/%a/", position - c); end = end->next = malloc(sizeof(struct mirror_st)); if(end == NULL) err(1, "malloc failed.\n"); end->next = NULL; position = 0; num = 0; } } } mTemp1 = start; while(mTemp1->next != end) mTemp1 = mTemp1->next; free(end->countryTitle); free(end); end = mTemp1; end->next = NULL; mTemp1 = start; while(mTemp1->next != NULL) { mTemp2 = mTemp1; do { if(!strcmp(mTemp1->mirror, mTemp2->next->mirror)) { mTemp3 = mTemp2->next; mTemp2->next = mTemp3->next; free(mTemp3->countryTitle); free(mTemp3->installPath); free(mTemp3->mirror); free(mTemp3); } mTemp2 = mTemp2->next; if(mTemp2 == NULL) break; } while( mTemp2->next != NULL ); mTemp1 = mTemp1->next; } int mirrorNum = 0; mTemp1 = start; while(mTemp1 != NULL) { ++mirrorNum; mTemp1 = mTemp1->next; } struct mirror_st ** Array; if( (Array = calloc(mirrorNum, sizeof(struct mirror_st *))) == NULL) err(1, "calloc failed."); mTemp1 = start; for(c = 0; c < mirrorNum; ++c) { Array[c] = mTemp1; mTemp1 = mTemp1->next; } qsort(Array, mirrorNum, sizeof(struct mirror_st *), country_cmp); fclose (input); close(sedToParent[0]); i = kevent(kqProc, NULL, 0, ke, 2, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { printf("timed out.\n"); kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); return -1; } else if(i == 1) { printf("timed out.\n"); if( ( (int *)ke->udata ) == &sedPid) kill(ftpPid, SIGKILL); else kill(sedPid, SIGKILL); return -1; } close(kq); close(kqProc); kqProc = kqueue(); if (kqProc == -1) err(1, "kqProc!"); int pid; for(c = 0; c < mirrorNum; ++c) { mTemp1 = Array[c]; position = 0; num = -1; printf("\n%d : %s : %s\n", (mirrorNum - c) - 1, mTemp1->countryTitle, mTemp1->mirror); pid = fork(); if(pid == (pid_t) 0) { /* ping child */ execl("/usr/bin/ftp", "ftp", "-Vmo", "/dev/null", mTemp1->mirror, NULL); err(1, "execl() failed\n"); } EV_SET(ke, pid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, NULL); if (kevent(kqProc, ke, 1, NULL, 0, NULL) == -1) err(1, "kevent register fail."); mTemp1->diff = 0; struct timeval tv_start, tv_end; gettimeofday(&tv_start, NULL); skip: i = kevent(kqProc, NULL, 0, ke, 1, &timeout); if (i == -1) err(1, "kevent"); if(i == 0) { printf("\n"); kill(pid, SIGKILL); mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0; // Wait until ftp() is dead before continuing. goto skip; } if( ke->data == 0 ) { gettimeofday(&tv_end, NULL); mTemp1->diff = getTimeDiff(tv_start, tv_end); } else if( mTemp1->diff == 0 ) mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0 + 1; } qsort(Array, mirrorNum, sizeof(struct mirror_st *), ftp_cmp); for(c = mirrorNum - 1; c >= 0; --c) { printf("%d : %s:\n\t%s: %f\n\n", c, Array[c]->countryTitle, Array[c]->installPath, Array[c]->diff); } char *buf; int lines; int totalLength = 0; for(position = 0; position < 8; ++position) totalLength += strlen("installpath += \n") + strlen(Array[position]->installPath); if( (input = fopen("/etc/pkg.conf", "r")) == NULL ) { buf = (char*)malloc(totalLength); if(buf == NULL) err(1, "malloc failed.\n"); c = strlcpy(buf, "installpath = ", totalLength); c += strlcpy(buf + c, Array[0]->installPath, totalLength - c); for(position = 1; position < 8; ++position) { c += strlcpy(buf + c, "\ninstallpath += ", totalLength - c); c += strlcpy(buf + c, Array[position]->installPath, totalLength - c); } } else { fseek(input, 0, SEEK_END); num = ftell(input); fseek(input, 0, SEEK_SET); lines = 0; buf = (char*)malloc(num + totalLength); if(buf == NULL) err(1, "malloc failed.\n"); fread(buf, 1, num, input); fclose(input); for(c = 0; c < num; ++c) { if(buf[c] == '\n') ++lines; } position = 0; int copy = 0; for(c = 0; c < num; ++c) { if(buf[c] == '\n') { --lines; position = 0; } else if(position++ == 0) { if( !strncmp(buf + c, "installpath", 11) ) { if(lines) { while(buf[++c] != '\n'); --lines; position = 0; continue; } else break; } } buf[copy++] = buf[c]; } //~ buf[copy] = '\0'; num += totalLength; c = copy + strlcpy(buf + copy, "installpath = ", num - copy); c += strlcpy(buf + c, Array[0]->installPath, num - c); for(position = 1; position < 8; ++position) { c += strlcpy(buf + c, "\ninstallpath += ", num - c); c += strlcpy(buf + c, Array[position]->installPath, num - c); } } input = fopen("/etc/pkg.conf", "w"); if(input != NULL) { fwrite(buf, 1, c, input); fclose(input); printf("\n\nEdit out all PKG_PATH environment exports and run \"unset PKG_PATH\".\n\n"); printf("/etc/pkg.conf:\n%s\n", buf); } else { printf("\n\nThis could have been the contents of /etc/pkg.conf (run as superuser):\n"); printf("%s\n", buf); } return 0; }
