pkg_ping [-s timeout] [-n maximum_mirrors_written] It scrapes each mirror's location and URL from openbsd.org/ftp.html and tests the package repository with the version and architecture of the machine. It kills the ftp() and sed() functions it calls from C if it takes too long by using kqueue. It calls uname as well and I put kqueue on it too, in case there is a chance uname can be called and stall like ftp.
After install, it can write download mirrors to /etc/pkg.conf. I want to enable the user to write down one or many mirrors as has been calculated by timing the download of the nearly 700 KB SHA256 file from each mirror. I think that if pkg_add can't find a suitable mirror, pkg_ping could be called to find the fastest available mirror(s), especially if their mirror of choice goes down, or they put off upgrading so long that their mirror of choice deletes their system's repository. I think I'm done with it. It is absolutely a coincidence that it is 666 lines. I'm not changing it. -Luke N Small
/* * Copyright (c) 2016 Luke N. Small * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* Special thanks to Dan Mclaughlin for the ftp to sed idea * * ftp -o - http://www.openbsd.org/ftp.html | \ * sed -n \ * -e 's:</a>$::' \ * -e 's: <strong>\([^<]*\)<.*:\1:p' \ * -e 's:^\( [hfr].*\):\1:p' */ #define EVENT_NOPOLL #define EVENT_NOSELECT #include <stdio.h> #include <unistd.h> #include <stdlib.h> #include <err.h> #include <errno.h> #include <sys/types.h> #include <sys/event.h> #include <signal.h> #include <string.h> struct mirror_st { char * countryTitle; char * mirror; char * installPath; double diff; struct mirror_st * next; }; int ftp_cmp (const void * a, const void * b) { struct mirror_st ** one = (struct mirror_st **)a; struct mirror_st ** two = (struct mirror_st **)b; if( (*one)->diff < (*two)->diff ) return -1; if( (*one)->diff > (*two)->diff ) return 1; return 0; } int country_cmp (const void * a, const void * b) { struct mirror_st ** one = (struct mirror_st **)a; struct mirror_st ** two = (struct mirror_st **)b; // list the USA mirrors first, it will subsort correctly int8_t temp = !strncmp("USA", (*one)->countryTitle, 3); if(temp != !strncmp("USA", (*two)->countryTitle, 3)) { if(temp) return -1; return 1; } return strcmp( (*one)->countryTitle, (*two)->countryTitle ) ; } double getTimeDiff(struct timeval a, struct timeval b) { long sec; long usec; sec = b.tv_sec - a.tv_sec; usec = b.tv_usec - a.tv_usec; if (usec < 0) { --sec; usec += 1000000; } return sec + ((double)usec / 1000000.0); } void manpage(char * a) { errx(1, "%s [-s timeout] [-n maximum_mirrors_written]", a); } int main(int argc, char *argv[]) { pid_t ftpPid, sedPid, unamePid; int ftpToSed[2]; int sedToParent[2]; int unameToParent[2]; char unameR[5], unameM[20], Char; int i; double S = 7; int position, num, c, N = 5000; FILE *input; if(argc > 1) { if(argc % 2 == 0) manpage(argv[0]); position = 0; while(++position < argc) { if(strlen(argv[position]) != 2) manpage(argv[0]); if(!strcmp(argv[position], "-s")) { ++position; c = -1; i = 0; while((Char = argv[position][++c]) != '\0') { if(Char == '.') ++i; if( ((Char < '0' || Char > '9') && Char != '.') || i > 1 ) { if(Char == '-') errx(1, "No negative numbers."); errx(1, "Incorrect floating point format."); } } errno = 0; strtod(argv[position], NULL); if(errno == ERANGE) err(1, NULL); if((S = strtod(argv[position], NULL)) > 100000000.0) errx(1, "The argument should less than or equal to 100000000"); } else if(!strcmp(argv[position], "-n")) { ++position; if(strlen(argv[position]) > 3) errx(1, "Integer should be less than or equal to 3 digits long."); c = -1; N = 0; while((Char = argv[position][++c]) != '\0') { if( Char < '0' || Char > '9' ) { if(Char == '.') errx(1, "No decimal points."); if(Char == '-') errx(1, "No negative numbers."); errx(1, "Incorrect integer format."); } N = N * 10 + (int)(Char - '0'); } } else manpage(argv[0]); } } struct kevent ke[2]; struct timespec timeout; timeout.tv_sec = (int)S; timeout.tv_nsec = (int)( (S - (double)timeout.tv_sec) * 1000000000 ); int kq = kqueue(); if (kq == -1) errx(1, "kq!"); if(pipe(unameToParent) == -1) err(1, NULL); // "uname -rm" returns version and architecture like: "5.8 amd64\n" to standard out unamePid = fork(); if(unamePid == (pid_t) 0) { /* uname child */ close(unameToParent[0]); dup2(unameToParent[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/uname","/usr/bin/uname", "-rm", NULL); errx(1, "uname execl() failed."); } if(unamePid == -1) err(1, NULL); close(unameToParent[1]); EV_SET(ke, unameToParent[0], EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, NULL); if (kevent(kq, ke, 1, NULL, 0, NULL) == -1) { kill(unamePid, SIGKILL); errx(1, "kevent register fail."); } i = kevent(kq, NULL, 0, ke, 1, &timeout); if (i == -1) { kill(unamePid, SIGKILL); errx(1, "kevent uname"); } if(i == 0) { kill(unamePid, SIGKILL); errx(1, "uname timed out."); } input = fdopen (unameToParent[0], "r"); if(input == NULL) err(1, NULL); num = 0; position = -1; while ((c = getc(input)) != EOF) { if(num == 0) { if(++position >= 5) errx(1, "unameR[] got too long!"); if(c != ' ') unameR[position] = c; else { unameR[position] = '\0'; num = 1; position = -1; } } else { if(++position >= 20) errx(1, "unameM[] got too long!"); if(c != '\n') unameM[position] = c; else { unameM[position] = '\0'; break; } } } fclose (input); close(unameToParent[0]); if(pipe(ftpToSed) == -1) err(1, NULL); ftpPid = fork(); if(ftpPid == (pid_t) 0) { /*ftp child*/ close(ftpToSed[0]); dup2(ftpToSed[1], STDOUT_FILENO); /*attaching to pipe(s)*/ execl("/usr/bin/ftp","ftp", "-Vo", "-", "http://www.openbsd.org/ftp.html";, NULL); errx(1, "ftp execl() failed."); } if(ftpPid == -1) err(1, NULL); close(ftpToSed[1]); if(pipe(sedToParent) == -1) err(1, NULL); sedPid = fork(); if(sedPid == (pid_t) 0) { /* sed child */ close(sedToParent[0]); dup2(ftpToSed[0], STDIN_FILENO); /*attaching to pipe(s)*/ dup2(sedToParent[1], STDOUT_FILENO); execl("/usr/bin/sed","sed","-n", "-e", "s:</a>$::", "-e", "s:\t<strong>\\([^<]*\\)<.*:\\1:p", "-e", "s:^\\(\t[hfr].*\\):\\1:p", NULL); kill(ftpPid, SIGKILL); errx(1, "sed execl() failed."); } if(sedPid == -1) { kill(ftpPid, SIGKILL); err(1, NULL); } EV_SET(ke, sedToParent[0], EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, NULL); if (kevent(kq, ke, 1, NULL, 0, NULL) == -1) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "sedToParent kevent register fail."); } close(ftpToSed[0]); /*close pipe(s) child attached to*/ close(sedToParent[1]); i = kevent(kq, NULL, 0, ke, 1, &timeout); if (i == -1) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "kevent, timeout may be too large"); } if(i == 0) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "timed out fetching openbsd.org/ftp.html."); } input = fdopen (sedToParent[0], "r"); if(input == NULL) err(1, NULL); char line[300]; num = 0; position = 0; struct mirror_st *start, *end, *mTemp1, *mTemp2, *mTemp3; start = end = malloc(sizeof(struct mirror_st)); if(start == NULL) errx(1, "malloc failed."); start->next = NULL; while((c = getc (input)) != EOF) { if(position >= 300) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "line[] got too long!"); } if(num == 0) { if( c != '\n' ) line[position++] = c; else { // there is a space before the newline that is eliminated line[position++] = '\0'; end->countryTitle = malloc(position); if(end->countryTitle == NULL) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "malloc failed."); } strlcpy(end->countryTitle, line, position); position = 0; num = 1; } } else { if(position == 0) { if( (c != 'h') && (c != 'f') && (c != 'r') ) continue; else if(c == 'r') break; else if(c == 'f') { // changes ftp listings to http. ftp.html says they can be either line[position++] = 'h'; line[position++] = 't'; continue; } } if( c != '\n' ) line[position++] = c; else { line[position++] = '\0'; position += num = strlen(unameR) + 10 + strlen(unameM) + 7; end->mirror = malloc(position); if(end->mirror == NULL) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "malloc failed."); } c = strlcpy(end->mirror, line, position); c += strlcpy(end->mirror + c, unameR, position - c); c += strlcpy(end->mirror + c, "/packages/", position - c); c += strlcpy(end->mirror + c, unameM, position - c); strlcpy(end->mirror + c, "/SHA256", position - c); position += 15 - num; end->installPath = malloc(position); if(end->installPath == NULL) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "malloc failed."); } c = strlcpy(end->installPath, line, position); strlcpy(end->installPath + c, "%c/packages/%a/", position - c); end = end->next = malloc(sizeof(struct mirror_st)); if(end == NULL) { kill(ftpPid, SIGKILL); kill(sedPid, SIGKILL); errx(1, "malloc failed."); } end->next = NULL; position = 0; num = 0; } } } fclose (input); // this is to prevent possible segfault in the next sequence. if(start == end) errx(1, "Too few entries found."); mTemp1 = start; while(mTemp1->next != end) mTemp1 = mTemp1->next; free(end->countryTitle); free(end); mTemp1->next = NULL; // Eliminate redundant mirrors (no longer need the 'end' value) mTemp1 = start; while(mTemp1->next != NULL) { mTemp2 = mTemp1; do { if(!strcmp(mTemp1->mirror, mTemp2->next->mirror)) { mTemp3 = mTemp2->next; mTemp2 = mTemp2->next = mTemp3->next; free(mTemp3->countryTitle); free(mTemp3->installPath); free(mTemp3->mirror); free(mTemp3); if(mTemp2 == NULL) break; } else mTemp2 = mTemp2->next; } while( mTemp2->next != NULL ); mTemp1 = mTemp1->next; } int mirrorNum = 0; mTemp1 = start; while(mTemp1 != NULL) { ++mirrorNum; mTemp1 = mTemp1->next; } struct mirror_st ** Array; if( (Array = calloc(mirrorNum, sizeof(struct mirror_st *))) == NULL) errx(1, "calloc failed."); mTemp1 = start; for(c = 0; c < mirrorNum; ++c) { Array[c] = mTemp1; mTemp1 = mTemp1->next; } qsort(Array, mirrorNum, sizeof(struct mirror_st *), country_cmp); close(sedToParent[0]); for(c = 0; c < mirrorNum; ++c) { mTemp1 = Array[c]; printf("\n%d : %s : %s\n", (mirrorNum - c) - 1, mTemp1->countryTitle, mTemp1->mirror); ftpPid = fork(); if(ftpPid == (pid_t) 0) { /* ping child */ execl("/usr/bin/ftp", "ftp", "-Vmo", "/dev/null", mTemp1->mirror, NULL); errx(1, "ftp execl() failed."); } if(ftpPid == -1) err(1, NULL); EV_SET(ke, ftpPid, EVFILT_PROC, EV_ADD | EV_ONESHOT, NOTE_EXIT, 0, NULL); if (kevent(kq, ke, 1, NULL, 0, NULL) == -1) { kill(ftpPid, SIGKILL); errx(1, "kevent register fail."); } mTemp1->diff = 0; struct timeval tv_start, tv_end; gettimeofday(&tv_start, NULL); skip: i = kevent(kq, NULL, 0, ke, 1, &timeout); if (i == -1) { kill(ftpPid, SIGKILL); errx(1, "kevent"); } if(i == 0) { printf("\n"); kill(ftpPid, SIGKILL); mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0; // Loop until ftp() is dead. goto skip; } if( ke->data == 0 ) { gettimeofday(&tv_end, NULL); mTemp1->diff = getTimeDiff(tv_start, tv_end); } else if( mTemp1->diff == 0 ) mTemp1->diff = timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0 + 1; } qsort(Array, mirrorNum, sizeof(struct mirror_st *), ftp_cmp); printf("\n\n"); for(c = mirrorNum - 1; c >= 0; --c) { printf("%d : %s:\n\t%s : ", c, Array[c]->countryTitle, Array[c]->installPath); if(Array[c]->diff < timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0) printf("%f\n\n", Array[c]->diff); else if(Array[c]->diff == timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0) printf("Timeout\n\n"); else printf("Download Error\n\n"); } if(Array[0]->diff >= timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0) errx(1, "No mirrors found within timeout period."); char *buf; int lines; int totalLength = 0; int copy = 0; for(position = 0; position < mirrorNum; ++position) totalLength += strlen("installpath += \n") + strlen(Array[position]->installPath); if( (input = fopen("/etc/pkg.conf", "r")) == NULL ) { buf = (char*)malloc(totalLength + 1); if(buf == NULL) errx(1, "malloc failed."); } else { fseek(input, 0, SEEK_END); num = ftell(input); fseek(input, 0, SEEK_SET); lines = 0; buf = (char*)malloc(num + totalLength + 11 + 1); if(buf == NULL) errx(1, "malloc failed."); fread(buf, 1, num, input); fclose(input); for(c = 0; c < num; ++c) { if(buf[c] == '\n') ++lines; } position = 0; for(c = 0; c < num; ++c) { if(buf[c] == '\n') { if(--lines == 0 && position == 0) break; // get rid of the lonely newline at the end, if it exists position = 0; } else if(position++ == 0) { if( !strncmp(buf + c, "installpath", 11) ) { if(lines) { while(buf[++c] != '\n'); --lines; position = 0; continue; } else break; } } buf[copy++] = buf[c]; } totalLength += num; } copy += strlcpy(buf + copy, "installpath = ", totalLength - copy); copy += strlcpy(buf + copy, Array[0]->installPath, totalLength - copy); if(N < mirrorNum) mirrorNum = N; position = 1; while(position < mirrorNum) { // Eliminates dowload error and timeout mirrors if(Array[position]->diff >= timeout.tv_sec + (double)timeout.tv_nsec / 1000000000.0) break; copy += strlcpy(buf + copy, "\ninstallpath += ", totalLength - copy); copy += strlcpy(buf + copy, Array[position++]->installPath, totalLength - copy); } copy += strlcpy(buf + copy, "\n", totalLength - copy); input = fopen("/etc/pkg.conf", "w"); if(input != NULL) { fwrite(buf, 1, copy, input); fclose(input); printf("\n\nEdit out all PKG_PATH environment variable exports"); printf(" and run \"unset PKG_PATH\".\n\n/etc/pkg.conf:\n%s\n", buf); } else { printf("\n\nThis could have been the contents of /etc/pkg.conf (run as superuser):\n"); printf("%s\n", buf); } if(argc == 1) printf("%s [-s timeout] [-n maximum_mirrors_written]\n", argv[0]); return 0; }
