OK, I've finally come up with a deduping script. It's not perfect, in that I
didn't know what to do with duplicate nodes with the same version, so I just
left them alone, but it's a start. Hopefully, over time, it'll work out OK,
the idea being that as later versions of nodes are added, the older dupes will
drop out.
For this reason, I've modified the addnodes.sh script to just add everything in
noderefs.txt (after first deduping it) to seednodes.ref, and then deduping
seednodes.ref. The problem with the earlier version of addnodes.sh was that if
a node was already present in seednodes.ref, the one in noderefs.txt would
never get added, even if it was newer.
I've also cleaned up and "normalized" all the other scripts, cleaning up the
headers and adding usage tips, as well as made them safer by having them backup
files before modifying them.
Hope you'll find them useful.
Conrad (my God, it's nearly 5 a.m.!)
--
Conrad Sabatier <[EMAIL PROTECTED]> - "In Unix veritas"
#!/bin/sh
#
# versions.sh
#
# Show a count of the number of each unique node version in input file
#
# Author: dolphin
#
# Usage: versions.sh inputfile
if [ $# -ne 1 ]
then
echo "Usage: versions.sh inputfile"
exit 1
fi
grep ^version $1 | sort | uniq -c
#!/bin/sh
#
# Displays IP and version info for each node reference in a file
#
# Author: dolphin
#
# Usage: nodes.sh infile
if [ $# -ne 1 ]
then
echo "Usage: nodes.sh infile"
exit 1
fi
#function: read a single noderef from input, save to temp file
read_one_noderef()
{
rm -f /tmp/noderef.$$
while read line
do
echo $line >> /tmp/noderef.$$
if [ "$line" = "End" ]
then
break
fi
done
}
infile=$1
exec < $infile
i=0
while :
do
read_one_noderef
if [ ! -f /tmp/noderef.$$ ]
then
break #end of file
fi
physical_tcp=$(awk '/^physical.tcp=/ {split($0, a, "[=:]"); print a[2]}' /tmp/n
oderef.$$)
echo -n "physical.tcp: ${physical_tcp}:"
port=$(awk -F':' '/^physical.tcp=/ {print $2}' /tmp/noderef.$$)
echo -n "${port} "
version=$(awk -F',' '/^version=/ {print $4}' /tmp/noderef.$$)
echo "version: ${version}"
i=$((i + 1))
done
echo "$i node references found"
#!/bin/sh
#
# refresh_bookmarks.sh
#
# Keep freenet bookmarks refreshed
#
# Author: dolphin
#
# Usage: ./refresh_bookmarks.sh
#
# Put this script in your main freenet dir and run from there
# (expects to find freenet.conf and lib/freenet.jar in current dir)
# Uncomment the following line to skip searching the local store
# (better for improving your node's routing), or comment it out
# to use the local store
skipDS='--skipDS'
logLevel=error #adjust to taste
cd $(realpath $(dirname $0))
HTL=$(awk -F'=' '/maxHopsToLive=/ {print $2}' freenet.conf)
SleepOnFail=$(($(awk -F'=' '/failureTableTime=/ {print $2}' freenet.conf) / 500)
)
SleepOnSuccess=$((SleepOnFail / 2))
for key in $(awk -F'=' '/bookmarks\..*\.key=/ {print $2}' freenet.conf | sort -u
)
do
{ while :
do
echo "Refreshing $key"
if java -cp lib/freenet.jar freenet.client.cli.Main get $key \
--htl ${HTL} ${skipDS} --logLevel ${logLevel}
then
sleep ${SleepOnSuccess}
else
sleep ${SleepOnFail}
fi
done
} & #don't remove this ampersand here! Not a typo! :-)
sleep 600 #wait 10 minutes before invoking next instance
done &
#!/bin/sh
#
# dedupe.sh
#
# Dedupe nodes in input file (seednodes.ref or noderefs.txt)
#
# Author: dolphin
#
# Usage: ./dedupe.sh inputfile
#
# Put this script in your main freenet dir and run from there
# (companion script addnodes.sh expects to find it in current dir)
#
# Algorithm:
#
# Step 1: backup input file! :-)
#
# Step 2: read noderef records one at a time from input file,
# saving version info to a temp file for each unique IP
#
# Step 3: sort unique the version numbers in each IP's temp file
#
# Step 4: reread the input file one record at a time as above,
# saving any nodes whose version is found in its IP's
# temp file, discarding all others, writing output to
# a temporary nodes file
#
# Step 5: move temporary nodes file to original input file
#
# Not terribly elegant, but it gets the job one :-)
#
# One little bugaboo: if a node has only multiple
# references to a single version, none are deleted.
# Didn't know how else to handle this without more
# to go on. :-)
if [ $# -ne 1 ]
then
echo "Usage: ./dedupe.sh infile"
exit 1
fi
infile=$1
cp ${infile} ${infile}.bak #save a copy just in case!
outfile=/tmp/$(basename ${infile}).$$
rm -f /tmp/node.* ${outfile}
#
#function: read a single noderef from input, save to temp file
#
read_one_noderef()
{
rm -f /tmp/noderef.$$
while read line
do
echo $line >> /tmp/noderef.$$
if [ "$line" = "End" ]
then
break
fi
done
}
# Collect node version info
i=0
while :
do
read_one_noderef
if [ ! -f /tmp/noderef.$$ ]
then
break #end of file
fi
physical_tcp=$(awk '/^physical.tcp=/ {split($0, a, "[=:]"); print a[2]}' /tmp/n
oderef.$$)
# echo -n "physical.tcp: ${physical_tcp}:"
port=$(awk -F':' '/^physical.tcp=/ { print $2 }' /tmp/noderef.$$)
# echo -n "${port} "
version=$(awk -F',' '/^version=/ {print $4}' /tmp/noderef.$$)
# echo "version: ${version}"
echo "${version}" >> /tmp/node.${physical_tcp}
i=$((i + 1))
done < $infile
echo "$i node references found in $infile"
echo "Sorting versions..."
#
# unique-ify version info
#
for f in /tmp/node.*
do
sort -urn -o $f $f
n=$(wc -l $f | awk '{print $1}')
if [ $n -gt 1 ]
then
rm -f ${f}.tmp
highest_version=0
# important! file *must* be sorted in reverse
# for the following loop to work properly
while read version
do
if [ $version -gt $highest_version ]
then
highest_version=$version
echo "$version" >> ${f}.tmp
fi
done < $f
mv ${f}.tmp $f
fi
done
echo "Deduping..."
i=0
while :
do
read_one_noderef
if [ ! -f /tmp/noderef.$$ ]
then
break #end of file
fi
physical_tcp=$(awk '/^physical.tcp=/ {split($0, a, "[=:]"); print a[2]}' /tmp/n
oderef.$$)
# echo -n "physical.tcp: ${physical_tcp}:"
port=$(awk -F':' '/^physical.tcp=/ { print $2 }' /tmp/noderef.$$)
# echo -n "${port} "
version=$(awk -F',' '/^version=/ {print $4}' /tmp/noderef.$$)
# echo "version: ${version}"
if grep -q $version /tmp/node.${physical_tcp}
then
# echo "Saving"
cat /tmp/noderef.$$ >> $outfile
i=$((i + 1))
fi
done < $infile
mv $outfile $infile
echo "Saved $i deduped nodes to $infile"
rm -f /tmp/node.*
#!/bin/sh
#
# cleannodes.sh
#
# Remove a range of node references (versions) from input file
#
# Author: dolphin
#
# Usage: cleannodes.sh infile minExclude maxExclude
if [ $# -ne 3 ]
then
echo "Usage: cleannodes.sh infile minExclude maxExclude"
exit 1
fi
infile=$1
outfile=/tmp/$(basename $infile).$$
minExclude=$2
maxExclude=$3
if [ $minExclude -gt $maxExclude ]
then
echo "Minimum exclude version must be <= maximum"
exit 1
fi
cp $infile ${infile}.bak #always backup! :-)
rm -f $outfile
# show version stats for input file
echo
echo "Input versions (file $infile):"
grep ^version $infile | sort | uniq -c
echo
#function: read a single noderef from input, save to temp file
read_one_noderef()
{
rm -f /tmp/noderef.$$
while read line
do
echo $line >> /tmp/noderef.$$
if [ "$line" = "End" ]
then
break
fi
done
}
while :
do
read_one_noderef
if [ ! -f /tmp/noderef.$$ ]
then
break #end of file
fi
version=$(awk -F',' '/^version=/ {print $4}' /tmp/noderef.$$)
echo -n version $version
if [ $version -ge $minExclude -a $version -le $maxExclude ]
then
echo ", discarding node"
rm -f /tmp/noderef.$$
else
echo ", keeping node"
cat /tmp/noderef.$$ >> $outfile
fi
done < $infile
mv $outfile $infile
# show version stats for output file
echo
echo "Output versions (file $infile):"
grep ^version $infile | sort | uniq -c
echo
rm -f /tmp/noderef.$$
#!/bin/sh
#
# addnodes.sh
#
# Add nodes from current routing table to seednodes.ref
# Dedupes seednodes.ref after adding nodes
#
# Author: dolphin
#
# Usage: ./addnodes.sh
#
# Put this script in your main freenet dir and run from there
# (expects to find noderefs.txt, seednodes.ref and dedupe.sh
# in current dir)
#
# Requires: wget
cd $(realpath $(dirname $0))
cp noderefs.txt noderefs.txt.bak
wget -q -O noderefs.txt http://localhost:8888/servlet/nodestatus/noderefs.txt
./dedupe.sh noderefs.txt
cat noderefs.txt >> seednodes.ref
./dedupe.sh seednodes.ref
_______________________________________________
Support mailing list
[EMAIL PROTECTED]
http://news.gmane.org/gmane.network.freenet.support
Unsubscribe at http://dodo.freenetproject.org/cgi-bin/mailman/listinfo/support
Or mailto:[EMAIL PROTECTED]
