This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new e6fff74 ATLAS-4011: Dockerfile: updated to support storing data
(solr, berkeleydb, kafka) on host directory
e6fff74 is described below
commit e6fff74807c24a06aea1cb7666bd07da9d1100ed
Author: Jayendra Parab <[email protected]>
AuthorDate: Tue Nov 3 00:00:26 2020 +0530
ATLAS-4011: Dockerfile: updated to support storing data (solr, berkeleydb,
kafka) on host directory
---
dev-support/atlas-docker/Dockerfile.atlas | 4 ++++
dev-support/atlas-docker/README.md | 4 ++--
dev-support/atlas-docker/data/.gitignore | 1 +
dev-support/atlas-docker/docker-compose.atlas.yml | 2 ++
dev-support/atlas-docker/scripts/atlas.sh | 4 +++-
distro/src/bin/atlas_config.py | 28 +++++++++++++++++++++--
distro/src/bin/atlas_start.py | 2 +-
distro/src/bin/atlas_stop.py | 2 +-
distro/src/test/python/scripts/TestMetadata.py | 2 +-
9 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/dev-support/atlas-docker/Dockerfile.atlas
b/dev-support/atlas-docker/Dockerfile.atlas
index e2dea77..76e5c59 100644
--- a/dev-support/atlas-docker/Dockerfile.atlas
+++ b/dev-support/atlas-docker/Dockerfile.atlas
@@ -26,8 +26,12 @@ RUN tar xfz /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz
--directory=/opt/ && \
rm -f /tmp/apache-atlas-${ATLAS_VERSION}-bin.tar.gz && \
mkdir -p /var/run/atlas && \
mkdir -p /var/log/atlas && \
+ mkdir -p /home/atlas/data && \
rm -rf ${ATLAS_HOME}/logs && \
ln -s /var/log/atlas ${ATLAS_HOME}/logs && \
+ ln -s /home/atlas/data ${ATLAS_HOME}/data && \
chown -R atlas:atlas ${ATLAS_HOME}/ /var/run/atlas/ /var/log/atlas/
+VOLUME /home/atlas/data
+
ENTRYPOINT [ "/home/atlas/scripts/atlas.sh" ]
diff --git a/dev-support/atlas-docker/README.md
b/dev-support/atlas-docker/README.md
index de64447..37caa3a 100644
--- a/dev-support/atlas-docker/README.md
+++ b/dev-support/atlas-docker/README.md
@@ -67,9 +67,9 @@ Docker files in this folder create docker images and run them
to build Apache At
This might take about 10 minutes to complete.
- 5.8. Execute following command to install and run Atlas services in a
container:
+ 5.5. Execute following command to install and run Atlas services in a
container:
- docker run -it -d --name atlas --hostname atlas.example.com -p
21000:21000 atlas
+ docker run -it -d --name atlas --hostname atlas.example.com -p
21000:21000 -v $(pwd)/data:/home/atlas/data atlas
This might take few minutes to complete.
diff --git a/dev-support/atlas-docker/data/.gitignore
b/dev-support/atlas-docker/data/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/dev-support/atlas-docker/data/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/dev-support/atlas-docker/docker-compose.atlas.yml
b/dev-support/atlas-docker/docker-compose.atlas.yml
index 92bd1ed..cfaf0d7 100644
--- a/dev-support/atlas-docker/docker-compose.atlas.yml
+++ b/dev-support/atlas-docker/docker-compose.atlas.yml
@@ -13,6 +13,8 @@ services:
tty: true
networks:
- atlas
+ volumes:
+ - ./data:/home/atlas/data
ports:
- "21000:21000"
depends_on:
diff --git a/dev-support/atlas-docker/scripts/atlas.sh
b/dev-support/atlas-docker/scripts/atlas.sh
index 0c4a454..7abb6f7 100755
--- a/dev-support/atlas-docker/scripts/atlas.sh
+++ b/dev-support/atlas-docker/scripts/atlas.sh
@@ -16,6 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -x
if [ ! -e ${ATLAS_HOME}/.setupDone ]
then
@@ -36,6 +37,7 @@ then
fi
su -c "cd ${ATLAS_HOME}/bin && ./atlas_start.py" atlas
+ATLAS_PID=`ps -ef | grep -v grep | grep -i "org.apache.atlas.Atlas" | awk
'{print $2}'`
# prevent the container from exiting
-/bin/bash
+tail --pid=$ATLAS_PID -f /dev/null
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index bda8e2a..7aa8196 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -50,7 +50,9 @@ MANAGE_EMBEDDED_CASSANDRA = "MANAGE_EMBEDDED_CASSANDRA"
MANAGE_LOCAL_ELASTICSEARCH = "MANAGE_LOCAL_ELASTICSEARCH"
SOLR_BIN = "SOLR_BIN"
SOLR_CONF = "SOLR_CONF"
+SOLR_HOME = "SOLR_HOME"
SOLR_PORT = "SOLR_PORT"
+SOLR_DIR = "SOLR_DIR"
DEFAULT_SOLR_PORT = "9838"
SOLR_SHARDS = "SOLR_SHARDS"
DEFAULT_SOLR_SHARDS = "1"
@@ -59,7 +61,7 @@ DEFAULT_SOLR_REPLICATION_FACTOR = "1"
ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP,
ATLAS_LOG, ATLAS_PID, ATLAS_CONF,
"ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP,
HBASE_CONF_DIR, SOLR_PORT, MANAGE_LOCAL_HBASE,
- MANAGE_LOCAL_SOLR, MANAGE_EMBEDDED_CASSANDRA,
MANAGE_LOCAL_ELASTICSEARCH]
+ MANAGE_LOCAL_SOLR, MANAGE_EMBEDDED_CASSANDRA,
MANAGE_LOCAL_ELASTICSEARCH, SOLR_HOME, SOLR_DIR]
IS_WINDOWS = platform.system() == "Windows"
ON_POSIX = 'posix' in sys.builtin_module_names
CONF_FILE="atlas-application.properties"
@@ -107,12 +109,21 @@ def hbaseConfDir(dir):
def zookeeperBinDir(dir):
return os.environ.get(SOLR_BIN, os.path.join(dir, "zk", BIN))
+def solrDir():
+ return os.environ.get(SOLR_DIR, os.path.join(atlasDir(), "solr"))
+
+def solrServerDir():
+ return os.path.join(solrDir(), "server")
+
def solrBinDir(dir):
return os.environ.get(SOLR_BIN, os.path.join(dir, "solr", BIN))
def elasticsearchBinDir(dir):
return os.environ.get(SOLR_BIN, os.path.join(dir, "elasticsearch", BIN))
+def solrHomeDir(dir):
+ return os.environ.get(SOLR_HOME, os.path.join(dir, "data", "solr"))
+
def solrConfDir(dir):
return os.environ.get(SOLR_CONF, os.path.join(dir, "solr",
CONFIG_SETS_CONF))
@@ -565,7 +576,7 @@ def start_elasticsearch(dir, logdir = None, wait=True):
sleep(6)
return processVal
-def run_solr(dir, action, zk_url = None, port = None, logdir = None,
wait=True):
+def run_solr(dir, action, zk_url = None, port = None, logdir = None,
wait=True, homedir = None):
solrScript = "solr"
@@ -583,6 +594,19 @@ def run_solr(dir, action, zk_url = None, port = None,
logdir = None, wait=True):
else:
cmd = [os.path.join(dir, solrScript), action, '-z', zk_url, '-p',
port]
+ if homedir is not None:
+ if not os.path.exists(homedir) :
+ os.makedirs(homedir)
+ #Copy solr.xml from installation directory to the solr home directory
+ srcSolrXmlPath = os.path.join(solrServerDir(), "solr", "solr.xml")
+ destSolrXmlPath = os.path.join(homedir, "solr.xml")
+ if not os.path.exists(destSolrXmlPath) :
+ print "solr.xml doesn't exist in " + homedir + ", copying from " +
srcSolrXmlPath
+ copyCmd = ["cp", srcSolrXmlPath, homedir]
+ runProcess(copyCmd, logdir, False, True)
+ cmd.append('-s')
+ cmd.append(homedir)
+
return runProcess(cmd, logdir, False, wait)
def create_solr_collection(dir, confdir, index, logdir = None, wait=True):
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index 54fd2bd..46e21ca 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -128,7 +128,7 @@ def main():
mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "start", logdir)
print "zookeeper started."
- mc.run_solr(mc.solrBinDir(atlas_home), "start",
mc.get_solr_zk_url(confdir), mc.solrPort(), logdir)
+ mc.run_solr(mc.solrBinDir(atlas_home), "start",
mc.get_solr_zk_url(confdir), mc.solrPort(), logdir, True,
mc.solrHomeDir(atlas_home))
print "solr started."
print "setting up solr collections..."
diff --git a/distro/src/bin/atlas_stop.py b/distro/src/bin/atlas_stop.py
index 400a7d2..6f5293e 100755
--- a/distro/src/bin/atlas_stop.py
+++ b/distro/src/bin/atlas_stop.py
@@ -68,7 +68,7 @@ def main():
# stop solr
if mc.is_solr_local(confdir):
- mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(),
None, True)
+ mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(),
None, True, mc.solrHomeDir(atlas_home))
if mc.is_zookeeper_local(confdir):
mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "stop")
diff --git a/distro/src/test/python/scripts/TestMetadata.py
b/distro/src/test/python/scripts/TestMetadata.py
index f1235f7..251de93 100644
--- a/distro/src/test/python/scripts/TestMetadata.py
+++ b/distro/src/test/python/scripts/TestMetadata.py
@@ -106,7 +106,7 @@ class TestMetadata(unittest.TestCase):
calls = [
call(['atlas_home/hbase/bin/hbase-daemon.sh', '--config',
'atlas_home/hbase/conf', 'start', 'master'],
'atlas_home/logs', False, True),
- call(['atlas_home/solr/bin/solr', 'start', '-z',
'localhost:9838', '-p', '9838'], 'atlas_home/logs',
+ call(['atlas_home/solr/bin/solr', 'start', '-z',
'localhost:9838', '-p', '9838', '-s', 'atlas_home/data/solr'],
'atlas_home/logs',
False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c',
'vertex_index', '-d',
'atlas_home/solr/server/solr/configsets/_default/conf',
'-shards', '1', '-replicationFactor',
