morningman commented on a change in pull request #1235: Support SSL authentication with Kafka in routine load job URL: https://github.com/apache/incubator-doris/pull/1235#discussion_r290552087
########## File path: fe/src/main/java/org/apache/doris/common/util/SmallFileMgr.java ########## @@ -0,0 +1,534 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.analysis.CreateFileStmt; +import org.apache.doris.analysis.DropFileStmt; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; + +import com.google.common.base.Strings; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Table; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +/* + * Author: Chenmingyu + * Date: May 29, 2019 + */ + +/* + * Manage some small files, such as certification file, public/private key used for some operations + */ +public class SmallFileMgr implements Writable { + public static final Logger LOG = LogManager.getLogger(SmallFileMgr.class); + + public static class SmallFile implements Writable { + public long dbId; + public String catalog; + public String name; + public long id; + public String content; + public long size; + public String md5; + public boolean isContent; + + private SmallFile() { + + } + + public SmallFile(Long dbId, String catalogName, String fileName, Long id, String content, long size, + String md5, boolean isContent) { + this.dbId = dbId; + this.catalog = catalogName; + this.name = fileName; + this.id = id; + this.content = content; + this.size = size; + this.md5 = md5.toLowerCase(); + this.isContent = isContent; + } + + public static SmallFile read(DataInput in) throws IOException { + SmallFile smallFile = new SmallFile(); + smallFile.readFields(in); + return smallFile; + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeLong(dbId); + Text.writeString(out, catalog); + Text.writeString(out, name); + out.writeLong(id); + Text.writeString(out, content); + out.writeLong(size); + Text.writeString(out, md5); + out.writeBoolean(isContent); + } + + @Override + public void readFields(DataInput in) throws IOException { + dbId = in.readLong(); + catalog = Text.readString(in); + name = Text.readString(in); + id = in.readLong(); + content = Text.readString(in); + size = in.readLong(); + md5 = Text.readString(in); + isContent = in.readBoolean(); + } + } + + public static class SmallFiles { + // file name -> file + private Map<String, SmallFile> files = Maps.newHashMap(); + + public SmallFiles() { + + } + + public Map<String, SmallFile> getFiles() { + return files; + } + + public void addFile(String fileName, SmallFile file) throws DdlException { + if (files.containsKey(fileName)) { + throw new DdlException("File " + fileName + " already exist"); + } + this.files.put(fileName, file); + } + + public SmallFile removeFile(String fileName) { + return files.remove(fileName); + } + + public SmallFile getFile(String fileName) { + return files.get(fileName); + } + + public boolean containsFile(String fileName) { + return files.containsKey(fileName); + } + } + + // db id -> catalog -> files + private Table<Long, String, SmallFiles> files = HashBasedTable.create(); + private Map<Long, SmallFile> idToFiles = Maps.newHashMap(); + + public SmallFileMgr() { + } + + public void createFile(CreateFileStmt stmt) throws DdlException { + String dbName = stmt.getDbName(); + Database db = Catalog.getCurrentCatalog().getDb(dbName); + if (db == null) { + throw new DdlException("Database " + dbName + " does not exist"); + } + downloadAndAddFile(db.getId(), stmt.getCatalogName(), stmt.getFileName(), + stmt.getDownloadUrl(), stmt.getChecksum(), stmt.isSaveContent()); + } + + public void dropFile(DropFileStmt stmt) throws DdlException { + String dbName = stmt.getDbName(); + Database db = Catalog.getCurrentCatalog().getDb(dbName); + if (db == null) { + throw new DdlException("Database " + dbName + " does not exist"); + } + removeFile(db.getId(), stmt.getCatalogName(), stmt.getFileName(), false); + } + + private void downloadAndAddFile(long dbId, String catalog, String fileName, String downloadUrl, String md5sum, + boolean saveContent) throws DdlException { + synchronized (files) { + if (idToFiles.size() >= Config.max_small_file_number) { + throw new DdlException("File number exceeds limit: " + Config.max_small_file_number); + } + } + + SmallFile smallFile = downloadAndCheck(dbId, catalog, fileName, downloadUrl, md5sum, saveContent); + + synchronized (files) { + if (idToFiles.size() >= Config.max_small_file_number) { + throw new DdlException("File number exceeds limit: " + Config.max_small_file_number); + } + + SmallFiles smallFiles = files.get(dbId, catalog); + if (smallFiles == null) { + smallFiles = new SmallFiles(); + files.put(dbId, catalog, smallFiles); + } + + smallFiles.addFile(fileName, smallFile); + idToFiles.put(smallFile.id, smallFile); + + Catalog.getCurrentCatalog().getEditLog().logCreateSmallFile(smallFile); + + LOG.info("finished to add file {} from url {}. current file number: {}", fileName, downloadUrl, + idToFiles.size()); + } + } + + public void replayCreateFile(SmallFile smallFile) { + synchronized (files) { + SmallFiles smallFiles = files.get(smallFile.dbId, smallFile.catalog); + if (smallFiles == null) { + smallFiles = new SmallFiles(); + files.put(smallFile.dbId, smallFile.catalog, smallFiles); + } + + try { + smallFiles.addFile(smallFile.name, smallFile); + idToFiles.put(smallFile.id, smallFile); + } catch (DdlException e) { + LOG.warn("should not happen", e); + } + } + } + + public void removeFile(long dbId, String catalog, String fileName, boolean isReplay) throws DdlException { + synchronized (files) { + SmallFiles smallFiles = files.get(dbId, catalog); + if (smallFiles == null) { + throw new DdlException("No such file in catalog: " + catalog); + } + SmallFile smallFile = smallFiles.removeFile(fileName); + if (smallFile != null) { + idToFiles.remove(smallFile.id); + + if (!isReplay) { + Catalog.getCurrentCatalog().getEditLog().logDropSmallFile(smallFile); + } + + LOG.info("finished to remove file {}. current file number: {}. is replay: {}", + fileName, idToFiles.size(), isReplay); + } else { + throw new DdlException("No such file: " + fileName); + } + } + } + + public void replayRemoveFile(SmallFile smallFile) { + try { + removeFile(smallFile.dbId, smallFile.catalog, smallFile.name, true); + } catch (DdlException e) { + LOG.error("should not happen", e); + } + } + + public boolean containsFile(long dbId, String catalog, String fileName) { + synchronized (files) { + SmallFiles smallFiles = files.get(dbId, catalog); + if (smallFiles == null) { + return false; + } + return smallFiles.containsFile(fileName); + } + } + + public SmallFile getSmallFile(long dbId, String catalog, String fileName, boolean needContent) + throws DdlException { + synchronized (files) { + SmallFiles smallFiles = files.get(dbId, catalog); + if (smallFiles == null) { + throw new DdlException("file does not exist with db: " + dbId + " and catalog: " + catalog); + } + SmallFile smallFile = smallFiles.getFile(fileName); + if (smallFile == null) { + throw new DdlException("File does not exist"); + } else if (needContent && !smallFile.isContent) { + throw new DdlException("File exists but not with content"); + } + return smallFile; + } + } + + public SmallFile getSmallFile(long fileId) { + synchronized (files) { + return idToFiles.get(fileId); + } + } + + private SmallFile downloadAndCheck(long dbId, String catalog, String fileName, + String downloadUrl, String md5sum, boolean saveContent) throws DdlException { + try { + URL url = new URL(downloadUrl); + // get file length + URLConnection urlConnection = url.openConnection(); + if (urlConnection instanceof HttpURLConnection) { + ((HttpURLConnection) urlConnection).setRequestMethod("HEAD"); + } + urlConnection.setReadTimeout(10000); // 10s + urlConnection.getInputStream(); + + int contentLength = urlConnection.getContentLength(); + if (contentLength == -1 || contentLength > Config.max_small_file_size_bytes) { + throw new DdlException("Failed to download file from url: " + url + ", invalid content length: " + contentLength); + } + + int bytesRead = 0; + String base64Content = null; + MessageDigest digest = MessageDigest.getInstance("MD5"); + if (saveContent) { + // download from url, and check file size + bytesRead = 0; + byte buf[] = new byte[contentLength]; + try (BufferedInputStream in = new BufferedInputStream(url.openStream())) { + while (bytesRead < contentLength) { + bytesRead += in.read(buf, bytesRead, contentLength - bytesRead); + } + + // check if there still has data(should not happen) + if (in.read() != -1) { + throw new DdlException("Failed to download file from url: " + url + + ", content length does not equals to actual file length"); + } + } + + if (bytesRead != contentLength) { + throw new DdlException("Failed to download file from url: " + url + + ", invalid read bytes: " + bytesRead + ", expected: " + contentLength); + } + + digest.update(buf, 0, bytesRead); + // encoded to base64 + base64Content = Base64.getEncoder().encodeToString(buf); + } else { + byte[] buf = new byte[4096]; + int tmpSize = 0; + try (BufferedInputStream in = new BufferedInputStream(url.openStream())) { + do { + tmpSize = in.read(buf); + if (tmpSize < 0) { + break; + } + digest.update(buf, 0, tmpSize); + bytesRead += tmpSize; + } while (true); + } + } + + // check md5sum if necessary + String checksum = Hex.encodeHexString(digest.digest()); + if (!Strings.isNullOrEmpty(md5sum)) { + if (!checksum.equals(md5sum)) { Review comment: ok ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
