This is an automated email from the ASF dual-hosted git repository. mykolabodnar pushed a commit to branch DLAB-515 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 44f4fbfc5f823f10a007466d76125ae4cbd01681 Merge: b45fecb d01594a Author: bodnarmykola <[email protected]> AuthorDate: Tue Aug 4 09:46:46 2020 +0300 Merge branch 'develop' into DLAB-515 .../scripts/jenkins/sonar.py | 36 ++ .../src/general/lib/os/debian/notebook_lib.py | 15 +- .../src/general/lib/os/fab.py | 14 +- .../src/general/lib/os/redhat/notebook_lib.py | 47 ++- .../general/scripts/os/install_additional_libs.py | 7 + .../aws/computational_resources/main/main.tf | 27 +- .../aws/computational_resources/main/variables.tf | 21 ++ .../terraform/aws/endpoint/main/bucket.tf | 54 +-- .../ssn-helm-charts/main/step-ca-chart/.helmignore | 21 ++ .../ssn-helm-charts/main/step-ca-chart/Chart.yaml | 23 +- .../main/step-ca-chart/templates/_helpers.tpl | 21 ++ .../main/step-ca-chart/templates/bootstrap.yaml | 47 ++- .../main/step-ca-chart/templates/ca.yaml | 203 ++++++------ .../main/step-ca-chart/templates/configmaps.yaml | 117 ++++--- .../main/step-ca-chart/templates/ingress.yaml | 63 ++-- .../main/step-ca-chart/templates/rbac.yaml | 23 +- .../main/step-ca-chart/templates/secrets.yaml | 21 ++ .../main/step-ca-chart/templates/service.yaml | 35 +- .../templates/tests/test-connection.yaml | 27 +- .../ssn-helm-charts/main/step-ca-chart/values.yaml | 25 +- .../terraform/bin/deploy/billing_app_aws.yml | 21 ++ .../terraform/bin/deploy/billing_azure.yml | 21 ++ .../terraform/bin/deploy/billing_gcp.yml | 21 ++ .../terraform/bin/deploy/endpoint_fab.py | 29 +- infrastructure-provisioning/terraform/bin/dlab.py | 20 +- .../terraform/bin/terraform-cli.py | 31 +- .../terraform/gcp/endpoint/provisioning.py | 25 +- .../terraform/gcp/main/main.tf | 29 +- .../terraform/gcp/main/variables.tf | 21 ++ .../terraform/gcp/modules/common/iam.tf | 25 +- .../terraform/gcp/modules/common/network.tf | 29 +- .../terraform/gcp/modules/common/variables.tf | 21 ++ .../terraform/gcp/modules/data_engine/instance.tf | 28 +- .../terraform/gcp/modules/data_engine/variables.tf | 21 ++ .../terraform/gcp/modules/dataproc/instance.tf | 83 +++-- .../terraform/gcp/modules/dataproc/variables.tf | 21 ++ .../terraform/gcp/modules/notebook/instance.tf | 23 +- .../terraform/gcp/modules/notebook/variables.tf | 21 ++ .../helm_charts/files/get_configmap_values.sh | 21 ++ .../modules/helm_charts/step-ca-chart/.helmignore | 21 ++ .../modules/helm_charts/step-ca-chart/Chart.yaml | 23 +- .../step-ca-chart/templates/_helpers.tpl | 21 ++ .../step-ca-chart/templates/bootstrap.yaml | 47 ++- .../helm_charts/step-ca-chart/templates/ca.yaml | 203 ++++++------ .../step-ca-chart/templates/configmaps.yaml | 117 ++++--- .../step-ca-chart/templates/ingress.yaml | 62 ++-- .../helm_charts/step-ca-chart/templates/rbac.yaml | 23 +- .../step-ca-chart/templates/secrets.yaml | 21 ++ .../step-ca-chart/templates/service.yaml | 35 +- .../templates/tests/test-connection.yaml | 27 +- .../modules/helm_charts/step-ca-chart/values.yaml | 25 +- .../dlab/login/resources/css/login.css | 19 ++ .../keycloak-theme/dlab/login/theme.properties | 33 +- integration-tests-cucumber/pom.xml | 19 ++ .../main/java/org/apache/dlab/dto/EndpointDTO.java | 19 ++ .../java/org/apache/dlab/mongo/MongoDBHelper.java | 19 ++ .../java/org/apache/dlab/util/JacksonMapper.java | 19 ++ .../java/org/apache/dlab/util/PropertyHelper.java | 19 ++ .../src/test/java/dlab/Constants.java | 19 ++ .../src/test/java/dlab/RunCucumberTest.java | 19 ++ .../src/test/java/dlab/endpoint/EndpointSteps.java | 19 ++ .../src/test/java/dlab/login/LoginSteps.java | 19 ++ .../src/test/resources/config.properties | 18 + .../src/test/resources/dlab/endpoint.feature | 18 + .../src/test/resources/dlab/login.feature | 18 + .../test_templates/rstudio/template_preparation.r | 17 + .../rstudio/template_visualization.r | 17 + pom.xml | 43 ++- .../main/java/com/epam/dlab/BillingService.java | 1 + .../billing-aws/src/main/resources/application.yml | 21 ++ .../azure/AzureInvoiceCalculationService.java | 2 - .../billing/azure/CalculateBillingService.java | 1 + .../dlab/billing/azure/config/SecurityConfig.java | 2 +- .../src/main/resources/application.yml | 21 ++ services/billing-gcp/billing.yml | 21 ++ .../epam/dlab/billing/gcp/conf/SecurityConfig.java | 4 +- .../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 3 +- .../billing/gcp/dao/impl/BigQueryBillingDAO.java | 11 +- .../dlab/billing/gcp/model/BillingHistory.java | 19 ++ .../gcp/repository/BillingHistoryRepository.java | 19 ++ .../dlab/billing/gcp/service/BillingService.java | 3 +- .../billing-gcp/src/main/resources/application.yml | 21 ++ .../dlab/auth/conf/AzureLoginConfiguration.java | 39 --- .../epam/dlab/auth/conf/GcpLoginConfiguration.java | 33 -- .../src/main/java/com/epam/dlab/MongoKeyWords.java | 30 -- .../epam/dlab/dto/base/project/ProjectResult.java | 19 ++ .../computational/UserComputationalResource.java | 4 + .../dlab/dto/exploratory/ExploratoryCreateDTO.java | 3 + .../dlab/dto/exploratory/ExploratoryGitCreds.java | 4 +- .../dlab/dto/exploratory/ExploratoryImageDTO.java | 1 + .../epam/dlab/dto/exploratory/LibInstallDTO.java | 5 + .../epam/dlab/dto/project/ProjectActionDTO.java | 19 ++ .../epam/dlab/dto/project/ProjectCreateDTO.java | 19 ++ .../main/java/com/epam/dlab/model/StringList.java | 35 +- .../java/com/epam/dlab/model/aws/ReportLine.java | 17 +- .../java/com/epam/dlab/model/library/Library.java | 6 + .../main/java/com/epam/dlab/util/FileUtils.java | 2 +- .../main/java/com/epam/dlab/util/ServiceUtils.java | 66 ++-- .../java/com/epam/dlab/util/UsernameUtils.java | 6 +- .../src/main/java/com/epam/dlab/ModuleBase.java | 22 +- .../src/main/java/com/epam/dlab/auth/UserInfo.java | 53 ++- .../java/com/epam/dlab/rest/contracts/EdgeAPI.java | 28 -- services/provisioning-service/pom.xml | 14 + .../DropwizardBearerTokenFilterImpl.java | 19 ++ .../epam/dlab/backendapi/core/MetadataHolder.java | 1 + .../dlab/backendapi/core/commands/CmdCommand.java | 1 + .../core/response/folderlistener/WatchItem.java | 20 +- .../response/handlers/ProjectCallbackHandler.java | 23 +- .../handlers/dao/FileSystemCallbackHandlerDao.java | 6 +- .../dlab/backendapi/resources/ProjectResource.java | 19 ++ .../resources/ProvisioningHealthCheckResource.java | 2 - .../backendapi/resources/aws/EdgeResourceAws.java | 7 +- .../resources/azure/EdgeResourceAzure.java | 7 +- .../backendapi/resources/gcp/EdgeResourceGcp.java | 7 +- .../dlab/backendapi/service/ProjectService.java | 19 ++ .../service/impl/CheckInactivityServiceImpl.java | 2 +- .../dlab/backendapi/service/impl/KeyService.java | 2 +- .../service/impl/ProjectServiceImpl.java | 47 ++- .../service/impl/aws/BucketServiceAwsImpl.java | 113 +++---- .../service/impl/azure/BucketServiceAzureImpl.java | 8 +- .../service/impl/gcp/BucketServiceGcpImpl.java | 10 +- .../dlab/process/builder/ProcessInfoBuilder.java | 17 +- services/self-service/entrypoint.sh | 21 ++ services/self-service/pom.xml | 11 + .../epam/dlab/backendapi/annotation/Project.java | 19 ++ .../backendapi/auth/KeycloakAuthenticator.java | 31 +- .../filters/DropwizardBearerTokenFilterImpl.java | 19 ++ .../dlab/backendapi/conf/CloudConfiguration.java | 21 +- .../backendapi/conf/KeycloakConfiguration.java | 19 ++ .../com/epam/dlab/backendapi/dao/AuditDAOImpl.java | 2 +- .../dao/{BackupDao.java => BackupDAO.java} | 2 +- .../dao/{BackupDaoImpl.java => BackupDAOImpl.java} | 2 +- .../dlab/backendapi/dao/ExploratoryLibDAO.java | 2 + ...xploratoryDao.java => ImageExploratoryDAO.java} | 2 +- ...ryDaoImpl.java => ImageExploratoryDAOImpl.java} | 16 +- .../com/epam/dlab/backendapi/dao/ProjectDAO.java | 19 ++ .../epam/dlab/backendapi/dao/ProjectDAOImpl.java | 23 +- .../com/epam/dlab/backendapi/dao/SecurityDAO.java | 44 ++- .../dao/{UserGroupDao.java => UserGroupDAO.java} | 4 +- ...UserGroupDaoImpl.java => UserGroupDAOImpl.java} | 2 +- .../dao/{UserRoleDao.java => UserRoleDAO.java} | 2 +- .../{UserRoleDaoImpl.java => UserRoleDAOImpl.java} | 5 +- .../dlab/backendapi/domain/CreateProjectDTO.java | 19 ++ .../epam/dlab/backendapi/domain/EndpointDTO.java | 1 - .../epam/dlab/backendapi/domain/ProjectDTO.java | 19 ++ .../dlab/backendapi/domain/ProjectEndpointDTO.java | 19 ++ .../backendapi/domain/UpdateProjectBudgetDTO.java | 19 ++ .../dlab/backendapi/domain/UpdateProjectDTO.java | 19 ++ .../dropwizard/bundles/DlabKeycloakBundle.java | 20 +- .../dropwizard/listeners/MongoStartupListener.java | 29 +- .../listeners/RestoreHandlerStartupListener.java | 2 +- .../backendapi/healthcheck/MongoHealthCheck.java | 2 +- .../interceptor/BudgetLimitInterceptor.java | 11 - .../epam/dlab/backendapi/modules/DevModule.java | 24 +- .../dlab/backendapi/modules/ProductionModule.java | 24 +- .../resources/ImageExploratoryResource.java | 2 +- .../backendapi/resources/KeycloakResource.java | 24 +- .../resources/LibExploratoryResource.java | 1 - .../dlab/backendapi/resources/ProjectResource.java | 33 +- .../resources/callback/GitCredsCallback.java | 3 - .../resources/callback/LibraryCallback.java | 2 - .../resources/callback/ProjectCallback.java | 19 ++ .../resources/callback/ReuploadKeyCallback.java | 2 - .../dlab/backendapi/resources/dto/KeysDTO.java | 25 +- .../resources/dto/ProjectActionFormDTO.java | 19 ++ .../com/epam/dlab/backendapi/roles/UserRole.java | 8 +- .../schedulers/billing/BillingScheduler.java | 2 +- .../endpoint/CheckEndpointStatusScheduler.java | 2 +- .../dlab/backendapi/service/AccessKeyService.java | 2 +- .../dlab/backendapi/service/EndpointService.java | 19 ++ .../backendapi/service/ExternalLibraryService.java | 1 + .../dlab/backendapi/service/GuacamoleService.java | 20 ++ .../dlab/backendapi/service/KeycloakService.java | 22 ++ .../backendapi/service/KeycloakServiceImpl.java | 30 +- .../dlab/backendapi/service/ProjectService.java | 41 ++- .../backendapi/service/ReuploadKeyService.java | 3 +- .../dlab/backendapi/service/SecurityService.java | 19 ++ .../backendapi/service/SecurityServiceImpl.java | 19 ++ .../dlab/backendapi/service/SystemInfoService.java | 1 + .../epam/dlab/backendapi/service/TagService.java | 20 ++ .../dlab/backendapi/service/TagServiceImpl.java | 21 +- .../backendapi/service/UserRoleServiceImpl.java | 4 +- .../backendapi/service/impl/BackupServiceImpl.java | 4 +- .../service/impl/BillingServiceImpl.java | 44 +-- .../backendapi/service/impl/BucketServiceImpl.java | 17 +- .../service/impl/ComputationalServiceImpl.java | 3 +- .../service/impl/EndpointServiceImpl.java | 107 +++--- .../service/impl/ExploratoryServiceImpl.java | 15 +- .../service/impl/GuacamoleServiceImpl.java | 25 +- .../service/impl/ImageExploratoryServiceImpl.java | 38 +-- .../impl/InfrastructureTemplateServiceImpl.java | 10 +- .../service/impl/MavenCentralLibraryService.java | 6 +- .../service/impl/ProjectServiceImpl.java | 138 ++++---- .../service/impl/UserGroupServiceImpl.java | 19 +- .../servlet/guacamole/GuacamoleServlet.java | 37 ++- .../epam/dlab/backendapi/util/CSVFormatter.java | 24 +- .../epam/dlab/backendapi/util/KeycloakUtil.java | 43 ++- .../validation/annotation/LibNameValid.java | 3 +- .../src/main/resources/webapp/browserslist | 23 +- .../manage-environment-dilog.component.html | 13 +- .../manage-environment-dilog.component.scss | 3 +- .../manage-environment-dilog.component.ts | 5 +- .../management-grid/management-grid.component.html | 4 +- .../management-grid/management-grid.component.scss | 62 +++- .../management/management.component.html | 2 +- .../project-list/project-list.component.scss | 11 +- .../administration/project/project.component.html | 2 +- .../src/app/core/interceptors/error.interceptor.ts | 3 +- .../services/applicationServiceFacade.service.ts | 9 + .../webapp/src/app/core/services/audit.service.ts | 19 ++ .../app/core/services/bucket-browser.service.ts | 19 ++ .../src/app/core/services/healthStatus.service.ts | 8 + .../core/services/librariesInstallation.service.ts | 1 - .../audit/audit-grid/audit-grid.component.html | 73 +++-- .../audit/audit-grid/audit-grid.component.scss | 79 +++-- .../audit/audit-grid/audit-grid.component.ts | 7 +- .../audit-toolbar/audit-toolbar.component.html | 4 +- .../audit-toolbar/audit-toolbar.component.scss | 76 +---- .../audit/audit-toolbar/audit-toolbar.component.ts | 34 +- .../src/app/reports/audit/audit.component.ts | 8 +- .../src/app/reports/audit/filter-audit.model.ts | 19 ++ .../reporting-grid/reporting-grid.component.html | 93 +++++- .../reporting-grid/reporting-grid.component.scss | 116 +++++-- .../reporting-grid/reporting-grid.component.ts | 49 ++- .../reporting/toolbar/toolbar.component.html | 2 +- .../reporting/toolbar/toolbar.component.scss | 76 ----- .../reports/reporting/toolbar/toolbar.component.ts | 33 +- .../bucket-browser/bucket-browser.component.html | 5 +- .../bucket-browser/bucket-data.service.ts | 37 ++- .../buckets-tree/bucket-tree.component.html | 21 +- .../buckets-tree/bucket-tree.component.scss | 19 +- .../buckets-tree/bucket-tree.component.ts | 19 ++ .../folder-tree/folder-tree.component.html | 21 +- .../folder-tree/folder-tree.component.scss | 18 + .../folder-tree/folder-tree.component.ts | 19 ++ .../computational-resources-list.component.scss | 46 +-- .../cost-details-dialog.component.html | 3 +- .../cost-details-dialog.component.scss | 3 +- .../install-libraries.component.html | 121 ++----- .../install-libraries.component.scss | 113 ++++--- .../install-libraries.component.ts | 10 +- .../resources-grid/resources-grid.component.html | 20 +- .../resources-grid/resources-grid.component.scss | 362 +++++++++++---------- .../src/app/resources/resources.component.html | 49 +-- .../src/app/resources/resources.component.ts | 11 +- .../shared/form-controls/dropdowns.component.scss | 7 +- .../multi-level-select-dropdown.component.scss | 5 +- .../notification-dialog.component.ts | 5 +- .../src/app/shared/navbar/navbar.component.ts | 105 ++++-- .../src/app/webterminal/webterminal.component.html | 2 +- .../webapp/src/assets/styles/_dialogs.scss | 5 + .../webapp/src/assets/styles/_general.scss | 9 +- .../resources/webapp/src/assets/styles/_theme.scss | 196 ++++++++++- .../webapp/src/assets/styles/_variables.scss | 2 + .../src/main/resources/webapp/src/styles.scss | 7 + .../backendapi/resources/KeycloakResourceTest.java | 25 +- .../resources/LibExploratoryResourceTest.java | 3 +- .../backendapi/resources/ProjectResourceTest.java | 173 +++++----- .../epam/dlab/backendapi/resources/TestBase.java | 17 +- .../service/KeycloakServiceImplTest.java | 135 ++++---- .../service/UserRoleServiceImplTest.java | 18 +- .../service/impl/AccessKeyServiceImplTest.java | 66 ++++ .../service/impl/AuditServiceImplTest.java | 106 ++++++ .../service/impl/BackupServiceImplTest.java | 15 +- .../service/impl/BucketServiceImplTest.java | 262 +++++++++++++++ .../impl/ImageExploratoryServiceImplTest.java | 4 +- .../InfrastructureTemplateServiceBaseTest.java | 4 +- .../service/impl/UserGroupServiceImplTest.java | 42 +-- 268 files changed, 5487 insertions(+), 2088 deletions(-) diff --cc infrastructure-provisioning/src/general/lib/os/fab.py index 205b2e7,45cd5c7..2c20b43 --- a/infrastructure-provisioning/src/general/lib/os/fab.py +++ b/infrastructure-provisioning/src/general/lib/os/fab.py @@@ -54,12 -54,10 +54,12 @@@ def dataengine_dir_prepare(cluster_dir) def install_pip_pkg(requisites, pip_version, lib_group): status = list() - error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires" + error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|RuntimeError:|error:" try: if pip_version == 'pip3' and not exists('/bin/pip3'): - sudo('ln -s /bin/pip3.5 /bin/pip3') + for v in range(4, 8): + if exists('/bin/pip3.{}'.format(v)): + sudo('ln -s /bin/pip3.{} /bin/pip3'.format(v)) sudo('{} install -U pip=={} setuptools'.format(pip_version, os.environ['conf_pip_version'])) sudo('{} install -U pip=={} --no-cache-dir'.format(pip_version, os.environ['conf_pip_version'])) sudo('{} install --upgrade pip=={}'.format(pip_version, os.environ['conf_pip_version'])) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
