This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble.git

commit 91368fbb9cfb09eaad8ce62af38ee06e8d8b153c
Author: Daniel Gruno <humbed...@apache.org>
AuthorDate: Fri Mar 2 10:50:57 2018 +0100

    Rewrite DB structure
    
    This removes document types altogether and uses one
    index per type instead, with a unified type mapping.
    
    This will also raise the minimum requirement for ES to 6.x.
    Backwards compatibility will be added to the UI for older
    DBs.
---
 setup/mappings.json | 640 ++++++++++++++++------------------------------------
 setup/setup.py      |  86 +++++--
 2 files changed, 262 insertions(+), 464 deletions(-)

diff --git a/setup/mappings.json b/setup/mappings.json
index 4b3046c..95fdb1c 100644
--- a/setup/mappings.json
+++ b/setup/mappings.json
@@ -1,454 +1,200 @@
 {
     "mappings": {
-        "email": {
-            "properties": {
-                "@version": {
-                    "type": "long"
-                },
-                "address": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "date": {
-                    "type": "date",
-                    "store": true,
-                    "format": "yyyy/MM/dd HH:mm:ss"
-                },
-                "hash": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "id": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "organisation": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "sender": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "sourceID": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "ts": {
-                    "type": "long"
-                }
-            }
-        },
-        "useraccount": {
-            "properties": {
-                "verified": {
-                    "type": "boolean",
-                    "index": "not_analyzed"
-                },
-                "email": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "displayName": {
-                    "type": "string"
-                },
-                "id": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "defaultOrganisation": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "organisations": {
-                    "type": "string"
-                },
-                "ownerships": {
-                    "type": "string"
-                },
-                "password": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "request_id": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "tag": {
-                    "type": "string",
-                    "index": "not_analyzed"
-                },
-                "verified": {
-                    "type": "boolean"
-                }
-            }
-        },
-    "code_commit": {
+        "_doc": {
         "properties": {
-            "@version": {
-                "type": "long"
-            },
-            "author_email": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "author_name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "committer_email": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "committer_name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "date": {
-                "type": "date",
-                "store": true,
-                "format": "yyyy/MM/dd HH:mm:ss"
-            },
-            "deletions": {
-                "type": "long"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "insertions": {
-                "type": "long"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "source": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "ts": {
-                "type": "long"
-            },
-            "tsday": {
-                "type": "long"
-            },
-            "vcs": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
+          "@version": {
+            "type": "long"
+          },
+          "address": {
+            "type": "keyword"
+          },
+          "admins": {
+            "type": "text"
+          },
+          "assignee": {
+            "type": "keyword"
+          },
+          "author_email": {
+            "type": "keyword"
+          },
+          "author_name": {
+            "type": "keyword"
+          },
+          "authors": {
+            "type": "long"
+          },
+          "blank": {
+            "type": "long"
+          },
+          "category": {
+            "type": "keyword"
+          },
+          "changeDate": {
+            "format": "yyyy/MM/dd HH:mm:ss",
+            "store": true,
+            "type": "date"
+          },
+          "closed": {
+            "type": "double"
+          },
+          "closedDate": {
+            "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
+            "type": "date"
+          },
+          "comments": {
+            "type": "long"
+          },
+          "committer_email": {
+            "type": "keyword"
+          },
+          "committer_name": {
+            "type": "keyword"
+          },
+          "cost": {
+            "type": "double"
+          },
+          "created": {
+            "type": "double"
+          },
+          "creator": {
+            "type": "double"
+          },
+          "createdDate": {
+            "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis",
+            "type": "date"
+          },
+          "date": {
+            "format": "yyyy/MM/dd HH:mm:ss",
+            "store": true,
+            "type": "date"
+          },
+          "defaultOrganisation": {
+            "type": "keyword"
+          },
+          "default_branch": {
+            "type": "text"
+          },
+          "deletions": {
+            "type": "long"
+          },
+          "displayName": {
+            "type": "text"
+          },
+          "email": {
+            "type": "keyword"
+          },
+          "emails": {
+            "type": "long"
+          },
+          "exception": {
+            "type": "text"
+          },
+          "good": {
+            "type": "boolean"
+          },
+          "hash": {
+            "type": "keyword"
+          },
+          "id": {
+            "type": "keyword"
+          },
+          "insertions": {
+            "type": "long"
+          },
+          "issueCloser": {
+            "type": "keyword"
+          },
+          "issueCreator": {
+            "type": "keyword"
+          },
+          "key": {
+            "type": "keyword"
+          },
+          "labels": {
+            "type": "text"
+          },
+          "loc": {
+            "type": "long"
+          },
+          "name": {
+            "type": "keyword"
+          },
+          "organisation": {
+            "type": "keyword"
+          },
+          "organisations": {
+            "type": "text"
+          },
+          "ownerships": {
+            "type": "text"
+          },
+          "password": {
+            "type": "keyword"
+          },
+          "request_id": {
+            "type": "keyword"
+          },
+          "sender": {
+            "type": "keyword"
+          },
+          "shash": {
+            "type": "keyword"
+          },
+          "source": {
+            "type": "keyword"
+          },
+          "sourceID": {
+            "type": "keyword"
+          },
+          "sourceURL": {
+            "type": "keyword"
+          },
+          "status": {
+            "type": "keyword"
+          },
+          "subject": {
+            "type": "text"
+          },
+          "sync": {
+            "type": "double"
+          },
+          "tag": {
+            "type": "keyword"
+          },
+          "text": {
+            "type": "text"
+          },
+          "time": {
+            "type": "double"
+          },
+          "title": {
+            "type": "text"
+          },
+          "topics": {
+            "type": "long"
+          },
+          "ts": {
+            "type": "long"
+          },
+          "tsday": {
+            "type": "long"
+          },
+          "type": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword"
+          },
+          "vcs": {
+            "type": "keyword"
+          },
+          "verified": {
+            "type": "boolean"
+          },
+          "years": {
+            "type": "double"
+          }
         }
-    },
-    "code_commit_unique": {
-        "properties": {
-            "@version": {
-                "type": "long"
-            },
-            "author_email": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "author_name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "committer_email": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "committer_name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "date": {
-                "type": "date",
-                "store": true,
-                "format": "yyyy/MM/dd HH:mm:ss"
-            },
-            "deletions": {
-                "type": "long"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "insertions": {
-                "type": "long"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "source": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "ts": {
-                "type": "long"
-            },
-            "tsday": {
-                "type": "long"
-            },
-            "vcs": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
-        }
-    },
-    "org": {
-        "properties": {
-            "admins": {
-                "type": "string"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "request_id": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
-        }
-    },
-    "mailstats": {
-        "properties": {
-            "authors": {
-                "type": "long"
-            },
-            "date": {
-                "type": "date",
-                "store": true,
-                "format": "yyyy/MM/dd HH:mm:ss"
-            },
-            "emails": {
-                "type": "long"
-            },
-            "hash": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "topics": {
-                "type": "long"
-            }
-        }
-    },
-    "mailtop": {
-        "properties": {
-            "date": {
-                "type": "date",
-                "store": true,
-                "format": "yyyy/MM/dd HH:mm:ss"
-            },
-            "emails": {
-                "type": "long"
-            },
-            "hash": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "shash": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "subject": {
-                "type": "string"
-            },
-            "ts": {
-                "type": "long"
-            }
-        }
-    },
-    "source": {
-        "properties": {
-            "default_branch": {
-                "type": "string"
-            },
-            "exception": {
-                "type": "string"
-            },
-            "good": {
-                "type": "boolean"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceURL": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sync": {
-                "type": "double"
-            },
-            "tag": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "type": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
-        }
-    },
-    "person": {
-        "properties": {
-            "@version": {
-                "type": "long"
-            },
-            "address": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "email": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "name": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
-        }
-    },
-    "issue": {
-        "properties": {
-            "assignee": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "changeDate": {
-                "type": "date",
-                "store": true,
-                "format": "yyyy/MM/dd HH:mm:ss"
-            },
-            "closed": {
-                "type": "double"
-            },
-            "closedDate": {
-                "type": "date",
-                "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis"
-            },
-            "comments": {
-                "type": "long"
-            },
-            "created": {
-                "type": "double"
-            },
-            "createdDate": {
-                "type": "date",
-                "format": "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis"
-            },
-            "id": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "issueCloser": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "issueCreator": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "key": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "status": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "title": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "url": {
-                "type": "string",
-                "index": "not_analyzed"
-            }
-        }
-    },
-    "evolution": {
-        "properties": {
-            "blank": {
-                "type": "long"
-            },
-            "comments": {
-                "type": "long"
-            },
-            "cost": {
-                "type": "double"
-            },
-            "loc": {
-                "type": "long"
-            },
-            "organisation": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "sourceID": {
-                "type": "string",
-                "index": "not_analyzed"
-            },
-            "time": {
-                "type": "double"
-            },
-            "years": {
-                "type": "double"
-            }
         }
     }
 }
-
-}
\ No newline at end of file
diff --git a/setup/setup.py b/setup/setup.py
index fb2bbe4..a691271 100644
--- a/setup/setup.py
+++ b/setup/setup.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 KIBBLE_VERSION = '0.1.0' # ABI/API compat demarcation.
-KIBBLE_DB_VERSION = 1 # First database revision
+KIBBLE_DB_VERSION = 2 # Second database revision
 
 import sys
 
@@ -147,26 +147,78 @@ def createIndex():
         retry_on_timeout=True
         )
 
+    es6 = True if int(es.info()['version']['number'].split('.')[0]) >= 6 else 
False
+    if not es6:
+        print("New Kibble installations require ElasticSearch 6.x or newer! 
You appear to be running %s!" % es.info()['version']['number'])
+        sys.exit(-1)
     # Check if index already exists
     if es.indices.exists(dbname):
         print("Error: ElasticSearch index '%s' already exists!" % dbname)
         sys.exit(-1)
 
-    print("Creating index " + dbname)
-
-    settings = {
-        "number_of_shards" :   shards,
-        "number_of_replicas" : replicas
-    }
-
- 
-    res = es.indices.create(index = dbname, body = {
-                "mappings" : mappings['mappings'],
-                "settings": settings
-            }
-        )
+    types = [
+        'api',
+        # ci_*: CI service stats
+        'ci_build',
+        'ci_queue',
+        # code_* + evolution + file_history: git repo stats
+        'code_commit',
+        'code_commit_unique',
+        'code_modification',
+        'evolution',
+        'file_history',
+        # forum_*: forum stats (SO, Discourse, Askbot etc)
+        'forum_post',
+        'forum_topic',
+        # GitHub stats
+        'ghstats',
+        # im_*: Instant messaging stats
+        'im_stats',
+        'im_ops',
+        'im_msg',
+        'issue',
+        'logstats',
+        # email, mail*: Email statitics
+        'email',
+        'mailstats',
+        'mailtop',
+        # organisation, view, source, publish: UI Org DB
+        'organisation',
+        'view',
+        'publish',
+        'source',
+        # stats: Miscellaneous stats
+        'stats',
+        # social_*: Twitter, Mastodon, Facebook etc
+        'social_follow',
+        'social_followers',
+        'social_follower',
+        'social_person',
+        # uisession, useraccount, message: UI user DB
+        'uisession',
+        'useraccount',
+        'message',
+        # person: contributor DB
+        'person',
+    ]
     
-    print("Index created! %s " % res)
+    for t in types:
+        iname = "%s_%s" % (dbname, t)
+        print("Creating index " + iname)
+    
+        settings = {
+            "number_of_shards" :   shards,
+            "number_of_replicas" : replicas
+        }
+    
+        
+        res = es.indices.create(index = iname, body = {
+                    "mappings" : mappings['mappings'],
+                    "settings": settings
+                }
+            )
+        
+    print("Indices created! %s " % res)
     
     salt = bcrypt.gensalt()
     pwd = bcrypt.hashpw(adminPass.encode('utf-8'), salt).decode('ascii')
@@ -185,8 +237,8 @@ def createIndex():
         'apiversion': KIBBLE_VERSION,           # Log current API version
         'dbversion': KIBBLE_DB_VERSION          # Log the database revision we 
accept (might change!)
     }
-    es.index(index=dbname, doc_type='useraccount', id = adminName, body = doc)
-    es.index(index=dbname, doc_type='api', id = 'current', body = dbdoc)
+    es.index(index=dbname + '_useraccount', doc_type = '_doc', id = adminName, 
body = doc)
+    es.index(index=dbname + '_api', doc_type = '_doc', id = 'current', body = 
dbdoc)
     print("Account created!")
 
 try:

-- 
To stop receiving notification emails like this one, please contact
humbed...@apache.org.

Reply via email to