This is an automated email from the ASF dual-hosted git repository. humbedooh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-foal.git
commit 4caefa86faeec65f6b5c3dc1f58b1d24c60b3809 Author: Daniel Gruno <[email protected]> AuthorDate: Mon Mar 29 15:00:38 2021 +0200 Simple utility for pushing previous failures to ES When archiving with --dumponfail, this utility program can help push failed documents into ES at a later stage. --- tools/push-failures.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tools/push-failures.py b/tools/push-failures.py new file mode 100755 index 0000000..3c15930 --- /dev/null +++ b/tools/push-failures.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Utility for retrying docs that we failed to index earlier. +""" + +import argparse +import json +import os +import plugins.elastic + +elastic = plugins.elastic.Elastic() + +parser = argparse.ArgumentParser(description="Command line options.") +parser.add_argument( + "--source", dest="dumpdir", help="Path to the directory containing the JSON documents that failed to index" +) + +args = parser.parse_args() + +dumpDir = args.dumpdir if args.dumpdir else "." + +print("Looking for *.json files in %s" % dumpDir) + +files = [f for f in os.listdir(dumpDir) if os.path.isfile(os.path.join(dumpDir, f)) and f.endswith(".json")] + +for f in files: + fpath = os.path.join(dumpDir, f) + print("Processing %s" % fpath) + with open(fpath, "r") as f: + ojson = json.load(f) + if "mbox" in ojson and "mbox_source" in ojson: + try: + mid = ojson["id"] + except KeyError: + mid = ojson["mbox"]["mid"] + elastic.index(index=elastic.db_mbox, id=mid, body=ojson["mbox"]) + + elastic.index(index=elastic.db_source, id=mid, body=ojson["mbox_source"]) + + if "attachments" in ojson and ojson["attachments"]: + for k, v in ojson["attachments"].items(): + elastic.index(index=elastic.db_attachment, id=k, body={"source": v}) + f.close() + os.unlink(fpath) +print("All done! Pushed %u documents to ES." % len(files))
