Repository: apex-site Updated Branches: refs/heads/master a81342e1c -> c88c954b3
APEXCORE-469 fix fetch-releases script to work with new dist structure Project: http://git-wip-us.apache.org/repos/asf/apex-site/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-site/commit/c88c954b Tree: http://git-wip-us.apache.org/repos/asf/apex-site/tree/c88c954b Diff: http://git-wip-us.apache.org/repos/asf/apex-site/diff/c88c954b Branch: refs/heads/master Commit: c88c954b313475422fd1bf0a27b6ec33fa11a7bd Parents: a81342e Author: Dean Lockgaard <[email protected]> Authored: Wed Apr 5 13:04:50 2017 -0700 Committer: Dean Lockgaard <[email protected]> Committed: Wed Apr 5 13:04:50 2017 -0700 ---------------------------------------------------------------------- README.md | 10 +- gulpfile.js | 304 +++++++++++++++++++++++++++++++---------------------- package.json | 1 + releases.json | 12 ++- 4 files changed, 192 insertions(+), 135 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-site/blob/c88c954b/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 8e3b89e..976d674 100644 --- a/README.md +++ b/README.md @@ -107,12 +107,8 @@ This way you make your change and refresh the page to see the effect immediately Updating Downloads Page ----------------------- -The instructions below to automatically update releases are currently broken: APEXCORE-469 - -Until that is resolved, update the file manually. To find the release date, use `git cat-file tag <tagname>` - The downloads page uses the content of `./releases.json` to populate the tables found there. -Care has been taken to automatically generate this releases.json file. To do so, run: +Care has been taken to automatically generate this `releases.json` file. To do so, run: ```bash ./node_modules/.bin/gulp fetch-releases @@ -122,9 +118,9 @@ git commit -m 'updated releases' This will do the following things: -1. Parses out the release versions available via the [ASF dist pages](https://dist.apache.org/repos/dist/release/incubator/apex). +1. Parses out the release versions available via the [ASF dist pages](https://dist.apache.org/repos/dist/release/apex). 2. Queries Github for these found release tags to find the date they were published to github -3. Writes to releases.json with release information. +3. Writes to `releases.json` with release information. Once you have committed the changes to `releases.json`, follow the steps to contributing steps to publish the site to go live. http://git-wip-us.apache.org/repos/asf/apex-site/blob/c88c954b/gulpfile.js ---------------------------------------------------------------------- diff --git a/gulpfile.js b/gulpfile.js index 61f23d2..cb4c3da 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -12,9 +12,10 @@ var fs = require('fs'); var async = require('async'); var _ = require('lodash'); var exec = require('child_process').exec; -var semver = require('semver'); +var semver = require('semver'); var naturalSort = require('javascript-natural-sort'); var dateFormat = require('dateformat'); +var Q = require('q'); // Constants var TEMP_PARTIAL_LOCATION = './.tmp/partials/'; @@ -72,7 +73,7 @@ gulp.task('html', ['md2html'], function() { // Render the files in pages gulp.src('./src/pages/*.html') - .pipe(handlebars({ + .pipe(handlebars({ nav: require('./navigation.json'), releases: require('./releases.json'), roadmap: require('./roadmap.json') @@ -130,10 +131,10 @@ gulp.task('fetch-roadmap', function(taskCb) { var projects = [ { key: 'core', name: 'APEXCORE', apiUrl: 'https://issues.apache.org/jira/rest/api/2/', url: 'https://issues.apache.org/jira/' }, { key: 'malhar', name: 'APEXMALHAR', apiUrl: 'https://issues.apache.org/jira/rest/api/2/', url: 'https://issues.apache.org/jira/' } - ]; + ]; // JQL terms are separated with AND/OR and parameters outside JQL are separated with & - // + // // Query to look up all APEXCORE and APEXMALHAR issues with label of roadmap // https://issues.apache.org/jira/rest/api/2/search?jql=project+in+(APEXCORE,APEXMALHAR)+AND+labels+in+(roadmap)+and+fixVersion+in+(EMPTY,unreleasedVersions())+ORDER+BY+key // @@ -160,9 +161,9 @@ gulp.task('fetch-roadmap', function(taskCb) { request({ url: requestUrl, json: true - }, + }, function(err, httpResponse, versions) { - + // Abort on error if (err) { console.log('Error when trying to request URL: ', requestUrl); @@ -186,9 +187,9 @@ gulp.task('fetch-roadmap', function(taskCb) { request.post({ url: project.apiUrl + 'search', json: apiRequest - }, + }, function(err, httpResponse, jiras) { - + // Abort on error if (err) { return cb(err); @@ -204,11 +205,11 @@ gulp.task('fetch-roadmap', function(taskCb) { var apiRequests = []; for (var i = 1; i < pageCount; i++) { - apiRequests[i-1] = _.extend({}, - apiRequest, - { - startAt: i * pageSize, - maxResults: pageSize + apiRequests[i-1] = _.extend({}, + apiRequest, + { + startAt: i * pageSize, + maxResults: pageSize } ); } @@ -219,9 +220,9 @@ gulp.task('fetch-roadmap', function(taskCb) { request.post({ url: project.apiUrl + 'search', json: apiPageRequest - }, + }, function(err, httpResponse, pageJiras) { - + // Abort on error if (err) { return pageCb(err); @@ -245,7 +246,7 @@ gulp.task('fetch-roadmap', function(taskCb) { } else { // Return with a new project object with jiras. cb is from async.map call above - cb(null, _.extend({}, project, { + cb(null, _.extend({}, project, { jiras: jiras.issues.sort(function(a,b) {return naturalSort(a.key, b.key); }), versions: unreleasedVersions })); @@ -268,7 +269,7 @@ gulp.task('fetch-roadmap', function(taskCb) { // Use the project key and provide associated arrays of matching jiras and versions projectResults.forEach(function(project) { - _.set(fileContents, project.key, + _.set(fileContents, project.key, { name: project.name, url: project.url, @@ -286,40 +287,107 @@ gulp.task('fetch-roadmap', function(taskCb) { }); // Creates releases.json file. -// -// 1. Requests page that lists release versions (https://dist.apache.org/repos/dist/release/incubator/apex[/malhar]) +// +// 1. Requests page that lists release versions (https://dist.apache.org/repos/dist/release/apex) // 2. Queries Github for release tags to find the date they were published to github // 3. Writes to releases.json with release information. -// +// gulp.task('fetch-releases', function(taskCb) { - // The base location for release listings - var distUrl = 'https://dist.apache.org/repos/dist/'; - // The release "targets", in this case meaning apex-core and apex-malhar var targets = [ - // NOTE: Once apex leaves incubation, this object should be changed to exclude "incubator" - { key: 'core.src', path: 'release/incubator/apex', repo: 'apex-core' }, - { key: 'malhar.src', path: 'release/incubator/apex/malhar', repo: 'apex-malhar' } + { key: 'core.src', repo: 'apex-core' }, + { key: 'malhar.src', repo: 'apex-malhar' } ]; - // For each target, get the releases - async.map(targets, function(target, cb) { + // Get contents for the page that lists the release versions + function getReleasePageLinks() { + var dfd = Q.defer(); - // Request the page that lists the release versions, - // e.g. https://dist.apache.org/repos/dist/release/incubator/apex - request(distUrl + target.path, function(err, response) { - - // Abort for error + // The base location for release listings + var distUrl = 'https://dist.apache.org/repos/dist/release/apex/'; + + request(distUrl, function(err, response) { + // Abort if tags not found or something bad happened with the git ls-remote command if (err) { - return cb(err); + dfd.reject(err); } + dfd.resolve(response.body); + }); + return dfd.promise; + } + + // Get tags and hashes for a repo's releases + function getRepoTags(repoName) { + var dfd = Q.defer(); + // Get the tags for the repo + var gitCommand = 'git ls-remote --tags "git://git.apache.org/' + repoName + '.git"'; + exec(gitCommand, function(err, stdout, stderr) { + // Abort if tags not found or something bad happened with the git ls-remote command + if (err || stderr) { + def.reject(err || stderr); + } + // Lines from ls-remote command look like [COMMIT_HASH]\trefs/tags/[TAG_NAME] + var lines = stdout.split('\n'); + for (var i = 0; i < lines.length; i++) { + if (lines[i] && lines[i].trim().length > 0) { + // console.log("Processing line[", i, "] : ", lines[i]); + lines[i] = lines[i].split('\t'); + if (lines[i][1]) { + lines[i][1] = lines[i][1].replace('refs/tags/v', ''); + } + } + } + dfd.resolve(lines); + }); + return dfd.promise; + } + + + // Get tags and hashes for all repos (apex-core and apex-malhar) + function getAllTargetReposTagsHashes() { - // Parse out the link names which are the - // available versions + return Q.all(targets.map(function(target) { + var dfd = Q.defer(); + getRepoTags(target.repo).then( + function(response) { + target.releases = response; + dfd.resolve(); + } + ); + return dfd.promise; + })); + } + + + // Get data for a single tag + function getTagDate(repoName, tagHash) { + var dfd = Q.defer(); + // Get info about the tag via its hash (found with the ls-remote command) + request({ + url: 'https://api.github.com/repos/apache/' + repoName + '/git/tags/' + tagHash, // Github API address + json: true, + headers: { 'User-Agent': 'apache' } + }, + function(err, response) { + // Abort if the commit could not be found + if (err) { + dfd.reject(err); + } + // Set the date from this information and resolve + dfd.resolve(Date.parse(response.body.tagger.date)); + } + ); + return dfd.promise; + } + + + // Start by getting links from the releases page + getReleasePageLinks().then( + function(response, err) { jsdom.env( - response.body, - function (err, window) { + response, + function(err, window) { // Query the DOM for all links in the list var releaseLinks = window.document.querySelectorAll('ul li a'); @@ -335,112 +403,98 @@ gulp.task('fetch-releases', function(taskCb) { // Create array of releases from this filtered NodeList var releases = releaseLinks.map(function(el) { + var releaseStr = el.innerHTML.trim(); + var repoName; + + // Set correct repo name + targets.forEach(function(tar) { + if (releaseStr.indexOf(tar.repo) !== -1) { + repoName = tar.repo; + } + }); + + // Extract release version and docs version + var releaseVersionText = el.innerHTML.trim(); + + // full release version + if (/([0-9]+\.[0-9]+\.[0-9]+((.*)([^\/]))*)/.test(releaseVersionText) === true) { + var releaseVersion = /([0-9]+\.[0-9]+\.[0-9]+((.*)([^\/]))*)/.exec(releaseVersionText)[1]; + } + + // strictly numbers for semantic version checking + if (/([0-9]+\.[0-9]+\.[0-9]+)/.test(releaseVersionText) === true) { + var releaseSemVer = /([0-9]+\.[0-9]+\.[0-9]+)/.exec(releaseVersionText)[1]; + } + var docsVersion = semver.major(releaseSemVer) + '.' + semver.minor(releaseSemVer); - // Trim the href attribute of leading "v" and trailing slash - var releaseVersion = el.innerHTML.trim().replace(/^v/, '').replace(/\/$/, ''); - var docsVersion = semver.major(releaseVersion) + '.' + semver.minor(releaseVersion); return { version: releaseVersion, docs: docsVersion, - // Add repo for use in async.each call below - repo: target.repo + repo: repoName }; }); - // Get the date for each release via the github API - async.each(releases, function(release, eachCb) { - - // Get the tags for the repo - var gitCommand = 'git ls-remote --tags "git://git.apache.org/' + release.repo + '.git"'; - exec(gitCommand, function(err, stdout, stderr) { + // Get tags and hashes for all target repos + getAllTargetReposTagsHashes().then(function(resp, err) { - // Abort if tags not found or something bad happened with the git ls-remote command - if (err || stderr) { - return eachCb(err || stderr); - } + if (err) { + console.log('ERROR', err); + } - // Lines from ls-remote command look like [COMMIT_HASH]\trefs/tags/[TAG_NAME] - var lines = stdout.split('\n'); - var tagHash; - - // Find hash for this release's tag - for (var i = 0; i < lines.length; i++) { - if (lines[i] && lines[i].trim().length > 0) { - // console.log("Processing line[", i, "] : ", lines[i]); - var parts = lines[i].split('\t'); - if (parts[1] && parts[1].replace(/^refs\/tags\/v?/, '') === release.version) { - tagHash = parts[0]; - break; + // Go through all releases + Q.all(releases.map(function(release) { + var def = Q.defer(); + + // go through target releases + // first, find the correct target + for (var i=0; i<targets.length; i++) { + if(targets[i].repo === release.repo) { + for(var j=0; j<targets[i].releases.length; j++) { + + // if target version matches release + if (targets[i].releases[j][1] === release.version) { + + // get tag date from remote repo + // arguments: repoName, tagHash + getTagDate(release.repo, targets[i].releases[j][0]).then(function(response) { + release.date=response; + def.resolve(); + }); + break; + } } + break; } } + return def.promise; + })) + + // After all tag dates are loaded + .then(function() { + // sort releases by date + releases.sort(function(a, b) { + return b.date - a.date; + }); - // Ensure we found one - if (!tagHash) { - return eachCb('Could not find tag from ls-remote command'); - } - - // Get info about the tag via its hash (found with the ls-remote command) - request({ - url: 'https://api.github.com/repos/apache/' + release.repo + '/git/tags/' + tagHash, // Github API address - json: true, - headers: { 'User-Agent': 'apache' } - }, function(err, response) { - - // Abort if the commit could not be found - if (err) { - return eachCb(err); - } - - // Set the date from the this information - release.date = Date.parse(response.body.tagger.date); - - // We're all done - eachCb(); + var fileContents = {}; + // Use the "key" to set core.src and malhar.src, etc. + targets.forEach(function(trg) { + // filter releases by repo + _.set(fileContents, trg.key, releases.filter(function(el) { + return el.repo === trg.repo; + })); }); - - }); - - }, function(err) { // callback for async.each(releases, ...) - // Abort if error occurred somewhere - if (err) { - return cb(err); - } + // Write the file to releases.json + fs.writeFile('./releases.json', JSON.stringify(fileContents, 0, 2)); - // Sort the releases by the date - releases.sort(function(a, b) { - return b.date - a.date; }); - - // Return with a new target object with releases. - // Note that this is the cb from the async.map call above - cb(null, _.extend({}, target, { releases: releases }) ); - }); - - } // end jsdom.env callback - ); // end jsdom.env - - }); // end request to the listing of this target - - - }, function(err, targetsWithVersions) { // this is the async.map(targets) callback - - // This will be written to releases.json - var fileContents = {}; - - // Use the "key" to set core.src and malhar.src, etc. - targetsWithVersions.forEach(function(trg) { - _.set(fileContents, trg.key, trg.releases); - }); - - // Write the file to releases.json - fs.writeFile('./releases.json', JSON.stringify(fileContents, 0, 2), taskCb); - - }); - + } + ); + } + ); }); // Watch for changes http://git-wip-us.apache.org/repos/asf/apex-site/blob/c88c954b/package.json ---------------------------------------------------------------------- diff --git a/package.json b/package.json index d1f46e2..40a8b0e 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "javascript-natural-sort": "^0.7.1", "jsdom": "3.1.2", "lodash": "^3.10.1", + "q": "^1.5.0", "request": "^2.65.0", "semver": "^5.1.0" } http://git-wip-us.apache.org/repos/asf/apex-site/blob/c88c954b/releases.json ---------------------------------------------------------------------- diff --git a/releases.json b/releases.json index 007cfd0..d53b02e 100644 --- a/releases.json +++ b/releases.json @@ -5,7 +5,7 @@ "version": "3.5.0", "docs": "3.5", "repo": "apex-core", - "date": 1481089442000 + "date": 1481394284000 }, { "version": "3.4.0", @@ -27,8 +27,14 @@ "version": "3.6.0", "docs": "3.6", "repo": "apex-malhar", - "date": 1480203354000 + "date": 1480713577000 + }, + { + "version": "3.5.0", + "docs": "3.5", + "repo": "apex-malhar", + "date": 1472969775000 } ] } -} +} \ No newline at end of file
