QWQyyy commented on issue #5429: URL: https://github.com/apache/openwhisk/issues/5429#issuecomment-1629290262
I figured out that this error might stem from long running functions, my data processing function behaves fine when processing a tenth of the size of the CIFAR10 dataset, but terminates at minutes when processing the full CIFAR10 dataset. Therefore, I guess that openwhisk has some bugs. On the surface, this bug is that when the function executes for more than 1 minute, it will cut off the client and make the function execute in the background, but the real situation is that it reports an error after running for several minutes. I think this may be a worthwhile question. @style95 Results of the:  No error action code: ```python import pandas as pd from torchvision import datasets, transforms import numpy as np from minio import Minio from minio.error import S3Error import io def main(args): # 转换器 transform_train = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]) transform_test = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.226, 0.224, 0.225))]) # 数据集路径 data_path = "/dataset/CIFAR10" # 创建与MinIO服务器的连接 client = Minio( "192.168.1.23:9000", access_key="CuZRDbhLVIIjfDp4M4p8", secret_key="tFDb4aoeSqlCB0bH6BoYfNvIWXqSCJQCZ6UD0Hd9", secure=False ) # 创建存储桶 bucket_name = "cifar" try: if not client.bucket_exists(bucket_name): client.make_bucket(bucket_name) print(f"存储桶 {bucket_name} 创建成功") else: print(f"存储桶 {bucket_name} 已存在") except S3Error as exc: print(f"存储桶 {bucket_name} 创建失败: {exc}") # 加载训练集和测试集 train_dataset = datasets.CIFAR10(root=data_path, train=True, transform=transform_train, download=False) test_dataset = datasets.CIFAR10(root=data_path, train=False, transform=transform_test, download=False) # 将训练集数据保存到CSV文件 train_data = [] for i, (image, label) in enumerate(train_dataset): image_data = np.array(image).tolist() # 转换为列表 train_data.append([image_data, label]) if i >= len(train_dataset) * 0.1: # 控制训练集样本数量 break train_df = pd.DataFrame(train_data, columns=['image', 'label']) train_csv_data = train_df.to_csv(index=False) # 将数据保存到MinIO存储中 try: data_bytes = train_csv_data.encode() # 将字符串转换为字节类型 data_stream = io.BytesIO(data_bytes) # 创建可读取的字节流 client.put_object(bucket_name, "train_dataset.csv", data_stream, len(data_bytes)) print(f"训练集数据已成功保存到MinIO存储，存储桶：{bucket_name}，对象：train_dataset.csv") except S3Error as exc: print(f"保存训练集数据到MinIO存储失败: {exc}") # 将测试集数据保存到CSV文件 test_data = [] for i, (image, label) in enumerate(test_dataset): image_data = np.array(image).tolist() # 转换为列表 test_data.append([image_data, label]) if i >= len(test_dataset) * 0.1: # 控制测试集样本数量 break test_df = pd.DataFrame(test_data, columns=['image', 'label']) test_csv_data = test_df.to_csv(index=False) # 将数据保存到MinIO存储中 try: data_bytes = test_csv_data.encode() # 将字符串转换为字节类型 data_stream = io.BytesIO(data_bytes) # 创建可读取的字节流 client.put_object(bucket_name, "test_dataset.csv", data_stream, len(data_bytes)) print(f"测试集数据已成功保存到MinIO存储，存储桶：{bucket_name}，对象：test_dataset.csv") except S3Error as exc: print(f"保存测试集数据到MinIO存储失败: {exc}") result = {"res": "666"} return result ``` and error action code: ```python import pandas as pd from torchvision import datasets, transforms import numpy as np from minio import Minio from minio.error import S3Error import io def main(args): # 转换器 transform_train = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]) transform_test = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.226, 0.224, 0.225))]) # 数据集路径 data_path = "/dataset/CIFAR10" # 创建与MinIO服务器的连接 client = Minio( "192.168.1.23:9000", access_key="CuZRDbhLVIIjfDp4M4p8", secret_key="tFDb4aoeSqlCB0bH6BoYfNvIWXqSCJQCZ6UD0Hd9", secure=False ) # 创建存储桶 bucket_name = "cifar" try: if not client.bucket_exists(bucket_name): client.make_bucket(bucket_name) print(f"存储桶 {bucket_name} 创建成功") else: print(f"存储桶 {bucket_name} 已存在") except S3Error as exc: print(f"存储桶 {bucket_name} 创建失败: {exc}") # 加载训练集和测试集 train_dataset = datasets.CIFAR10(root=data_path, train=True, transform=transform_train, download=False) test_dataset = datasets.CIFAR10(root=data_path, train=False, transform=transform_test, download=False) # 将训练集数据保存到CSV文件 train_data = [] for i, (image, label) in enumerate(train_dataset): image_data = np.array(image).tolist() # 转换为列表 train_data.append([image_data, label]) train_df = pd.DataFrame(train_data, columns=['image', 'label']) train_csv_data = train_df.to_csv(index=False) # 将数据保存到MinIO存储中 try: data_bytes = train_csv_data.encode() # 将字符串转换为字节类型 data_stream = io.BytesIO(data_bytes) # 创建可读取的字节流 client.put_object(bucket_name, "train_dataset.csv", data_stream, len(data_bytes)) print(f"训练集数据已成功保存到MinIO存储，存储桶：{bucket_name}，对象：train_dataset.csv") except S3Error as exc: print(f"保存训练集数据到MinIO存储失败: {exc}") # 将测试集数据保存到CSV文件 test_data = [] for i, (image, label) in enumerate(test_dataset): image_data = np.array(image).tolist() # 转换为列表 test_data.append([image_data, label]) test_df = pd.DataFrame(test_data, columns=['image', 'label']) test_csv_data = test_df.to_csv(index=False) # 将数据保存到MinIO存储中 try: data_bytes = test_csv_data.encode() # 将字符串转换为字节类型 data_stream = io.BytesIO(data_bytes) # 创建可读取的字节流 client.put_object(bucket_name, "test_dataset.csv", data_stream, len(data_bytes)) print(f"测试集数据已成功保存到MinIO存储，存储桶：{bucket_name}，对象：test_dataset.csv") except S3Error as exc: print(f"保存测试集数据到MinIO存储失败: {exc}") result = {"res": "666"} return result ``` The code that only differs from the reduced dataset -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
