jenkins-bot has submitted this change and it was merged.
Change subject: Clean up to repository scripts
......................................................................
Clean up to repository scripts
Lots of cleanups in here, might contain traces of bugs.
Bug: T87004
Change-Id: I67f475ca46c89609e3ac645ee4519bdca82849af
---
A bin/clupdate-gerrit-repo
A bin/clupdate-git-repo
M bin/repocommit
M bin/repocreate
M bin/repoupdate
M bin/update-reset-repo
6 files changed, 92 insertions(+), 220 deletions(-)
Approvals:
Nikerabbit: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/clupdate-gerrit-repo b/bin/clupdate-gerrit-repo
new file mode 120000
index 0000000..184ea4f
--- /dev/null
+++ b/bin/clupdate-gerrit-repo
@@ -0,0 +1 @@
+clupdate-git-repo
\ No newline at end of file
diff --git a/bin/clupdate-git-repo b/bin/clupdate-git-repo
new file mode 100755
index 0000000..eca1039
--- /dev/null
+++ b/bin/clupdate-git-repo
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+set -u
+
+DIRSCRIPT="`dirname \"$0\"`"
+DIRSCRIPT="`( cd \"$DIRSCRIPT\" && pwd )`"
+
+ME=`basename "$0"`
+
+URL=$1
+DIR=$2
+BRANCH=${3:-master}
+
+if [ -d "$DIR/.git" ]
+then
+ "$DIRSCRIPT/update-reset-repo" "$DIR" "" "$BRANCH"
+else
+ git clone "$URL" "$DIR" -b "$BRANCH"
+
+ if [ "$ME" = 'clupdate-gerrit-repo' ]
+ then
+ cd "$DIR"
+ git config user.name 'Translation updater bot'
+ git config user.email 'l10n-...@translatewiki.net'
+ git config gitreview.username 'l10n-bot'
+
+ git review -r origin -s
+ fi
+fi
diff --git a/bin/repocommit b/bin/repocommit
index 0891df5..4ddec10 100755
--- a/bin/repocommit
+++ b/bin/repocommit
@@ -9,37 +9,22 @@
DIR=$2
: ${DIR:=`pwd`}
source ${DIRSCRIPT}/findexportroot
-cd "$DIR"
echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> ${DIRLOG}
COMMITMSG="Localisation updates from https://translatewiki.net.";
-
gitCommit() {
- local dir=$1
- cd "$dir"
+ cd "$1"
git add .
git commit -m "$COMMITMSG"
-
- # Check if $2 is set. If not, push to origin master
- if [[ -z "$2" ]]
- then
- git push origin master
- else
- git push origin $2
- fi
-
- cd ..
+ git push origin "${2:-master}"
}
gitCommitGerrit() {
- local dir=$1
- cd $PROJECT
+ cd $1
git add .
- git commit -q -m "$COMMITMSG" && git fetch -q gerrit && git review -t
l10n || :
- mergeL10n-bot
- cd ..
+ git commit -q -m "$COMMITMSG" && git review -r origin -t l10n || :
}
mergeL10n-bot() {
@@ -74,16 +59,16 @@
for i in $GITPROJECTS; do
if [ "$i" = "$PROJECT" ]
then
- gitCommit "$PROJECT"
+ gitCommit "$DIR/$PROJECT"
if [ "$PROJECT" = "blockly" ]
then
- gitCommit blockly-games
+ gitCommit "$DIR/blockly-games"
fi
if [ "$PROJECT" = "osm" ]
then
- gitCommit potlatch2
+ gitCommit "$DIR/potlatch2"
fi
exit 0
@@ -100,18 +85,19 @@
for i in $GERRITPROJECTS; do
if [ "$i" = "$PROJECT" ]
then
- gitCommitGerrit "$PROJECT"
+ gitCommitGerrit "$DIR/$PROJECT"
+ mergeL10n-bot
exit 0
fi
done
if [ "$PROJECT" = "etherpad-lite" ]
then
- gitCommit "$PROJECT" develop
+ gitCommit "$DIR/$PROJECT" develop
elif [ "$PROJECT" = "fudforum" ]
then
- cd $PROJECT
+ cd "$DIR/$PROJECT"
svn add -q --force *
for i in `find . -name msg`
do
@@ -119,22 +105,19 @@
svn propset -q svn:eol-style native $i
done
svn commit --message "$COMMITMSG"
- cd ..
elif [ "$PROJECT" = "ihris" ]
then
- cd $PROJECT
for MODULE in $REPO_IHRIS_MODULES
do
- cd $MODULE
+ cd "$DIR/$PROJECT/$MODULE"
bzr add .
bzr commit -m "$COMMITMSG"
- cd ..
done
elif [ "$PROJECT" = "inaturalist" ]
then
- gitCommit "$PROJECT" i18n
+ gitCommit "$DIR/$PROJECT" i18n
elif [ "$PROJECT" = "mediawiki" ]
then
@@ -142,67 +125,48 @@
then echo "Add REPO_MEDIAWIKI_BRANCHES to REPOCONF"; exit 1
fi
- cd $PROJECT
for BRANCH in $REPO_MEDIAWIKI_BRANCHES
do
- cd "$BRANCH"
- git add .
- git commit -m "$COMMITMSG" && git fetch gerrit && git review -t
l10n || :
- cd ..
+ gitCommitGerrit "$DIR/$PROJECT/$BRANCH"
done
- cd ..
elif [ "$PROJECT" = "mediawiki-extensions" ]
then
- cd $PROJECT/extensions
- for i in `ls`
+ for i in `ls "$DIR/$PROJECT/extensions"`
do
echo "--> $i"
- cd $i
- git add .
- git commit -m "$COMMITMSG" && git fetch gerrit && git review -t
l10n || :
- cd ..
+ gitCommitGerrit "$DIR/$PROJECT/extensions/$i"
done
-
- cd ../..
mergeL10n-bot
elif [ "$PROJECT" = "mediawiki-skins" ]
then
- cd $PROJECT
- for i in `ls`
+ for i in `ls $DIR/$PROJECT`
do
echo "--> $i"
- cd $i
- git add .
- git commit -m "$COMMITMSG" && git fetch gerrit && git review -t
l10n || :
- cd ..
+ gitCommitGerrit "$DIR/$PROJECT/$i"
done
-
- cd ..
mergeL10n-bot
elif [ "$PROJECT" = "mifos" ]
then
- gitCommit "$PROJECT" develop
+ gitCommit "$DIR/$PROJECT" develop
elif [ "$PROJECT" = "nocc" ]
then
- cd $PROJECT
+ cd "$DIR/$PROJECT"
svn add --force * --auto-props --parents --depth infinity -q
svn commit --message "$COMMITMSG"
- cd ..
elif [ "$PROJECT" = "openimages" ]
then
- cd $PROJECT
+ cd "$DIR/$PROJECT"
svn add -q --force *.properties
svn propset -q svn:mime-type text/plain *.properties
svn propset -q svn:eol-style native *.properties
svn commit --message "$COMMITMSG"
- cd ..
else
echo "`basename $0`: Unknown project"
diff --git a/bin/repocreate b/bin/repocreate
index ffc2bdd..6ff1773 100755
--- a/bin/repocreate
+++ b/bin/repocreate
@@ -13,17 +13,11 @@
echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> ${DIRLOG}
-gitCreateGerrit() {
- if [ "$REPO_RW" = "yes" ]
- then
- pushd "$1"
- git config user.name 'Translation updater bot'
- git config user.email 'l10n-...@translatewiki.net'
- git config gitreview.username 'l10n-bot'
- git review -s
- popd
- fi
-}
+
+if [ "${REPO_RW:-no}" = "yes" ]
+then CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-gerrit-repo";
+else CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-git-repo";
+fi
checkVar() {
if [ -z "${!1}" ]
@@ -131,11 +125,9 @@
checkVar 'REPO_MEDIAWIKI'
checkVar 'REPO_MEDIAWIKI_BRANCHES'
mkdir -p $PROJECT
- cd $PROJECT
for BRANCH in $REPO_MEDIAWIKI_BRANCHES
do
- git clone "$REPO_MEDIAWIKI" "$BRANCH" -b "$BRANCH"
- gitCreateGerrit $BRANCH
+ "$CLUPDATE_GERRIT" "$REPO_MEDIAWIKI" "$DIR/$PROJECT/$BRANCH"
$BRANCH
done
elif [ "$PROJECT" = "mediawiki-extensions" ]
@@ -176,8 +168,7 @@
elif [ "$PROJECT" = "pywikibot" ]
then
checkVar 'REPO_PYWIKIBOT'
- git clone "$REPO_PYWIKIBOT" $PROJECT
- gitCreateGerrit $PROJECT
+ "$CLUPDATE_GERRIT" "$REPO_PYWIKIBOT" "$DIR/$PROJECT"
elif [ "$PROJECT" = "vicuna" ]
then
@@ -197,20 +188,17 @@
elif [ "$PROJECT" = "wikimania" ]
then
checkVar 'REPO_WIKIMANIA'
- git clone "$REPO_WIKIMANIA" $PROJECT
- gitCreateGerrit $PROJECT
+ "$CLUPDATE_GERRIT" "$REPO_WIKIMANIA" "$DIR/$PROJECT"
elif [ "$PROJECT" = "wikipedia-android" ]
then
checkVar 'REPO_WIKIPEDIAANDROID'
- git clone "$REPO_WIKIPEDIAANDROID" $PROJECT
- gitCreateGerrit $PROJECT
+ "$CLUPDATE_GERRIT" "$REPO_WIKIPEDIAANDROID" "$DIR/$PROJECT"
elif [ "$PROJECT" = "wikipedia-ios" ]
then
checkVar 'REPO_WIKIPEDIAIOS'
- git clone "$REPO_WIKIPEDIAIOS" $PROJECT
- gitCreateGerrit $PROJECT
+ "$CLUPDATE_GERRIT" "$REPO_WIKIPEDIAIOS" "$DIR/$PROJECT"
elif [ "$PROJECT" = "WikisourceMobile" ]
then
diff --git a/bin/repoupdate b/bin/repoupdate
index d43958e..2d59e05 100755
--- a/bin/repoupdate
+++ b/bin/repoupdate
@@ -15,61 +15,29 @@
echo "$(date --rfc-3339=seconds --utc) [$(whoami) at $DIR] $0 $@" >> $DIRLOG
gitupdate() {
- local dir=$1
- cd $dir
- git fetch -q --all
- if [ -z "$2" ]
- then
- git reset -q --hard origin/master
- else
- git reset -q --hard origin/$2
- fi
- git clean -q -f -d
- git pull -q
- cd ..
+ "$DIRSCRIPT/update-reset-repo" "$DIR" "$1" "$2"
}
-cloneGerritRepo() {
- local url=$1
- local name=$2
-
- git clone "$url" "$name"
-
- if [ "$REPO_RW" = "yes" ]
+processGroups() {
+ if [ -z "$REPO_RW" ]
then
- cd "$name"
-
- git config user.name 'Translation updater bot'
- git config user.email 'l10n-...@translatewiki.net'
- git config gitreview.username 'l10n-bot'
-
- if [ ! -e ".gitreview" ]
- then
- git remote add -f gerrit "$url"
- fi
-
- git review -s
-
- cd ..
+ sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet
--group="$1" &&
+ echo "Please check
https://translatewiki.net/wiki/Special:ManageMessageGroups";
fi
}
-# Ugly. Eventually, I think we should end up with a file similar to
-# mediawiki-extensions.txt where we can simply add a line like this:
-# repo=https://gerrit.wikimedia.org/r/p/oojs/ui.git
-# and then things magically happen based on the other configuration
-# settings.
-processHacks() {
- if [ ! -d "oojs-ui/.git" ]
- then
- cloneGerritRepo "https://gerrit.wikimedia.org/r/p/oojs/ui.git";
"oojs-ui"
- fi
-
- if [ ! -d "VisualEditorVisualEditor/.git" ]
- then
- cloneGerritRepo
"https://gerrit.wikimedia.org/r/p/VisualEditor/VisualEditor.git";
"VisualEditorVisualEditor"
- fi
+fetchReposFromGerrit() {
+ curl -s "https://gerrit.wikimedia.org/r/projects/?p=$1"; |
+ grep "$1" |
+ cut -d '"' -f2 |
+ cut -d '/' -f3 |
+ sort
}
+
+if [ "${REPO_RW:-no}" = "yes" ]
+then CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-gerrit-repo";
+else CLUPDATE_GERRIT="$DIRSCRIPT/clupdate-git-repo";
+fi
GITUPDATEPROJECTS="\
eol \
@@ -141,36 +109,8 @@
then echo "Add REPO_MEDIAWIKI_BRANCHES to REPOCONF"; exit 1
fi
- cd $PROJECT
- for BRANCH in $REPO_MEDIAWIKI_BRANCHES
- do
- if [ ! -d "$BRANCH" ]
- then
- echo "$BRANCH not yet present. Cloning..."
- git clone "$REPO_MEDIAWIKI" "$BRANCH" -b "$BRANCH"
- if [ "$REPO_RW" = "yes" ]
- then
- cd "$BRANCH"
- git config user.name 'Translation updater bot'
- git config user.email
'l10n-...@translatewiki.net'
- git config gitreview.username 'l10n-bot'
- git review -s
- fi
- else
- cd "$BRANCH"
- git fetch -q --all
- git reset -q --hard "origin/$BRANCH"
- git clean -q -f -d
- fi
- cd ..
- done
- cd ..
-
- wait
- if [ "$DIR" = "/resources/projects" ]
- then
- sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet ||
${DIRSCRIPT}/udpcast Message index creation failed &
- fi
+ echo "$REPO_MEDIAWIKI_BRANCHES" | tr ' ' '\n' |
+ xargs -P4 I___ "$CLUPDATE_GERRIT" "$REPO_MEDIAWIKI"
"$DIR/$PROJECT/___" "___"
elif [ "$PROJECT" = "mediawiki-extensions" ]
then
@@ -178,34 +118,14 @@
then echo "Add REPO_MWEXTGIT to REPOCONF"; exit 1
fi
- cd $PROJECT
- cd extensions
+ echo "$(fetchReposFromGerrit mediawiki/extensions/)" |
+ xargs -P4 -I___ "$CLUPDATE_GERRIT" "$REPO_MWEXTGIT/___.git"
"$DIR/$PROJECT/extensions/___"
- EXTENSIONS=$(curl -s
https://gerrit.wikimedia.org/r/projects/?p=mediawiki/extensions/ |grep
'mediawiki/extensions/' |cut -d '"' -f 2 |cut -d '/' -f3 | sort)
+ "$CLUPDATE_GERRIT" "https://gerrit.wikimedia.org/r/p/oojs/ui.git";
"$DIR/$PROJECT/extensions/oojs-ui" &
+ "$CLUPDATE_GERRIT"
"https://gerrit.wikimedia.org/r/p/VisualEditor/VisualEditor.git";
"$DIR/$PROJECT/extensions/VisualEditorVisualEditor" &
+ wait
- # Clone extension repos that do not exist yet.
- for EXTENSION in $EXTENSIONS
- do
- if [ ! -d "$EXTENSION/.git" ]
- then
- echo "Cloning extension $EXTENSION"
- cloneGerritRepo "$REPO_MWEXTGIT/$EXTENSION.git"
"$EXTENSION"
- fi
- done
-
- # This is where I disappoint Niklas a bit more...
- processHacks
- EXTENSIONS="$EXTENSIONS VisualEditorVisualEditor oojs-ui"
-
- # Update all repos (yes, there is duplication for repos that were just
cloned)
- echo $EXTENSIONS |
- xargs -n1 -P7 ${DIRSCRIPT}/update-reset-repo
"$DIR/$PROJECT/extensions"
-
- if [ -z "$REPO_RW" ]
- then
- sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet
--group=ext-* && echo "Please check
https://translatewiki.net/wiki/Special:ManageMessageGroups";
- sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet ||
${DIRSCRIPT}/udpcast Message index creation failed &
- fi
+ processGroups "ext-*"
elif [ "$PROJECT" = "mediawiki-skins" ]
then
@@ -213,29 +133,10 @@
then echo "Add REPO_MWSKINGIT to REPOCONF"; exit 1
fi
- cd $PROJECT
+ echo "$(fetchReposFromGerrit mediawiki/skins/)" |
+ xargs -P4 -I___ "$CLUPDATE_GERRIT" "$REPO_MWSKINGIT/___.git"
"$DIR/$PROJECT/___"
- SKINS=$(curl -s
https://gerrit.wikimedia.org/r/projects/?p=mediawiki/skins/ |grep
'mediawiki/skins/' |cut -d '"' -f 2 |cut -d '/' -f3 |sort)
-
- # Clone skin repos that do not exist yet.
- for SKIN in $SKINS
- do
- if [ ! -d "$SKIN/.git" ]
- then
- echo "Cloning skin $SKIN"
- cloneGerritRepo "$REPO_MWSKINGIT/$SKIN.git" "$SKIN"
- fi
- done
-
- # Update all repos (yes, there is duplication for repos that were just
cloned)
- echo $SKINS |
- xargs -n1 -P7 ${DIRSCRIPT}/update-reset-repo "$DIR/$PROJECT"
-
- if [ -z "$REPO_RW" ]
- then
- sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/processMessageChanges.php" --quiet
--group=mediawiki-skin-* && echo "Please check
https://translatewiki.net/wiki/Special:ManageMessageGroups";
- sudo -u betawiki php
"$WIKI/extensions/Translate/scripts/createMessageIndex.php" --quiet ||
${DIRSCRIPT}/udpcast Message index creation failed &
- fi
+ processGroups "mediawiki-skin-*"
elif [ "$PROJECT" = "nocc" ]
then
diff --git a/bin/update-reset-repo b/bin/update-reset-repo
index dafb734..900fb56 100755
--- a/bin/update-reset-repo
+++ b/bin/update-reset-repo
@@ -1,4 +1,5 @@
#!/bin/bash
+set -e
# Allow for passing a base path and repo folder name in two parameters
# or a single full path as parameter.
@@ -10,26 +11,14 @@
DIR=$1
fi
+BRANCH=${3:-master}
+
# Updates the repository in the current directory and resets it.
-# Does not retry in case of errors.
# Sets OUTPUT to the output of the git operations
# Sets CODE to the return value of the pipeline.
update_reset() {
- OUTPUT="$(git ls-remote --exit-code origin refs/heads/master)"
+ OUTPUT=$(git fetch -q && git reset -q --hard origin/$BRANCH && git
clean -q -f -d 2>&1 >&-)
CODE=$?
-
- case "$CODE" in
- 0)
- # Actual fetching, resetting, ...
- OUTPUT=$(git fetch -q --all && git reset -q --hard
origin/master && git clean -q -f -d 2>&1 >&-)
- CODE=$?
- ;;
- 2)
- # remote does not have refs/heads/master
- # No need to try fetching, resetting, ...
- CODE=0
- ;;
- esac
}
cd $DIR
--
To view, visit https://gerrit.wikimedia.org/r/217944
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I67f475ca46c89609e3ac645ee4519bdca82849af
Gerrit-PatchSet: 1
Gerrit-Project: translatewiki
Gerrit-Branch: master
Gerrit-Owner: Nikerabbit <niklas.laxst...@gmail.com>
Gerrit-Reviewer: Nikerabbit <niklas.laxst...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
