Hi Canh, Ack with minor comments.
Regards, Vu > -----Original Message----- > From: Canh Van Truong <canh.v.tru...@dektech.com.au> > Sent: Friday, June 14, 2019 10:21 PM > To: lennart.l...@ericsson.com; vu.m.ngu...@dektech.com.au > Cc: opensaf-devel@lists.sourceforge.net; Canh Van Truong > <canh.v.tru...@dektech.com.au> > Subject: [PATCH 1/1] log: Delete the older file without closing current log file > via admin operation [#3046] > > Improve the enhancement of ticket #3022, the user can rotate (delete) older > log > files without closing current log file. > > LOG add new parameter (numberOfFilesToRemove) to admin op command > with id = 2. > - If parameter exists, LOGD remove number of older file base on parameter > value > - If parameter is omitted, LOGD rotates log file normally. (closing current log > file, rotate log file if needed, create new log file) > > This helps to avoid creating empty log file or small log file size > --- > src/log/apitest/tet_LogOiOps.c | 272 +++++++++++++++++++++++++++++---- > -------- > src/log/logd/lgs_filehdl.cc | 4 +- > src/log/logd/lgs_imm.cc | 104 ++++++++++++---- > src/log/logd/lgs_stream.cc | 41 +++++-- > src/log/logd/lgs_stream.h | 3 +- > 5 files changed, 305 insertions(+), 119 deletions(-) > > diff --git a/src/log/apitest/tet_LogOiOps.c b/src/log/apitest/tet_LogOiOps.c > index 947e79664..25f53d3a2 100644 > --- a/src/log/apitest/tet_LogOiOps.c > +++ b/src/log/apitest/tet_LogOiOps.c > @@ -5521,95 +5521,201 @@ void admin_rotate_log_file(void) > } > } > > +// Verify that log and cfg files are rotated when the number of log/cfg files > +// reach the "saLogStreamMaxFilesRotated" value (ticket #3045) > // > -// Test case for verifying the admin operation rotates log file with > -// parameter fails > -// Steps: > -// 1. Create configuration app stream > -// 2. Verify command admin operation rotates log file with parameter fails > -// 3. Delete configuration app stream > +// Step1: Create and delete the app stream with > (saLogStreamMaxFilesRotated = 2) > +// 4 times > +// Step2: Verify that total is 4 cfg and log files (2 log files and 2 cfg files) > +void verRotatedLogCfgFile(void) > +{ > + char command1[MAX_DATA], command2[MAX_DATA]; > + const char *object_dn = > + "safLgStrCfg=verRotatedFile,safApp=safLogService"; > + const int max_file = 2; > + char num_files_c[10]; > + uint32_t num_files = 0; > + > + // Command to create configuration application stream > + sprintf(command1, "immcfg -c SaLogStreamConfig %s " > + "-a saLogStreamPathName=. " > + "-a saLogStreamFileName=verRotatedFile" > + " -a saLogStreamMaxFilesRotated=%d", > + object_dn, max_file); > + // Command to delete configuration application stream > + sprintf(command2, "immcfg -d %s", object_dn); > + > + // Create and delete the app stream 4 times. > + // Log/cfg are created during creating stream. > + for (int i = 0; i < max_file + 1; ++i) { > + int rc = systemCall(command1); > + if (rc == 0) { > + osaf_nanosleep(&kHundredMilliseconds); > + rc = systemCall(command2); > + osaf_nanosleep(&kOneSecond); > + } > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + } > + // Command to count number of log/cfg files on disk > + sprintf(command1, "find %s -type f -mmin -1 " > + "| egrep '%s.*\\.[log$\\|cfg$]' " > + "| wc -l | awk '{printf $1}'", > + log_root_path, "verRotatedFile"); > + > + FILE *fp = popen(command1, "r"); > + > + // Get number of cfg and log file > + while (fgets(num_files_c, sizeof(num_files_c) - 1, fp) != NULL) {}; > + pclose(fp); > + > + // Verify cfg/log files are rotated by checking > + // that there are totally 4 cfg and log files > + num_files = atoi(num_files_c); > + rc_validate(num_files, max_file * 2); > +} > + > +// Verify that log file is rotated via admin operation with admin id = 2 > +// and no parameter > // > -void admin_rotate_log_file_with_param(void) > +// Step1: Create and delete the app stream with > (saLogStreamMaxFilesRotated = 2) > +// step2: Do admin operation 2 times to rotate log file. > +// Two new log file are created > +// step3: Verify log file is rotated via admin operation by checking that > +// there are 2 log files on disk > +void verRotatedLogCfgFile2(void) > { > - int rc = 0; > + int rc; > char command[MAX_DATA]; > const char *object_dn = > - > "safLgStrCfg=admin_rotate_file1,safApp=safLogService"; > + "safLgStrCfg=verRotatedFile2,safApp=safLogService"; > + char num_files_c[10]; > + uint32_t num_files = 0; > > - // Create configuration application stream > - sprintf(command, "immcfg -c SaLogStreamConfig %s" > - " -a saLogStreamPathName=. " > - "-a saLogStreamFileName=admin_rotate_file1 ", > - object_dn); > + // Command to create configuration application stream > + sprintf(command, "immcfg -c SaLogStreamConfig %s " > + " -a saLogStreamPathName=. " > + "-a saLogStreamFileName=verRotatedFile2" > + " -a saLogStreamMaxFilesRotated=2", > + object_dn); > + // Create the app stream. > rc = systemCall(command); > - if (rc == 0) { > - // Admin operation opId = 2 with parameter > - sprintf(command, "immadm -o 2 -p " > - "saLogStreamSeverityFilter:SA_UINT32_T:7 %s > >" > - " /dev/null 2>&1 ", object_dn); > - rc = system(command); > - rc_validate(WEXITSTATUS(rc), 1); > - > - // Delete object > - sprintf(command, "immcfg -d %s", object_dn); > - systemCall(command); > - } else { > + if (rc != 0) { > rc_validate(rc, 0); > + return; > } > + > + // Do admin operation to rotate log file without the parameter > + // Two new log files are created. One oldest file is removed. > + sprintf(command, "immadm -o 2 %s", object_dn); > + for (int i = 0; i < 2; ++i) { > + osaf_nanosleep(&kOneSecond); > + rc = systemCall(command); > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + } > + > + // Command to count number of log files on disk > + sprintf(command, "find %s -type f -mmin -1 " > + "| egrep '%s.*\\.log$' " > + "| wc -l | awk '{printf $1}'", > + log_root_path, "verRotatedFile2"); > + FILE *fp = popen(command, "r"); > + // Get number of log files > + while (fgets(num_files_c, sizeof(num_files_c) - 1, fp) != NULL) {}; > + pclose(fp); > + > + // Command to delete configuration application stream > + sprintf(command, "immcfg -d %s", object_dn); > + // Close the application stream > + rc = systemCall(command); > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + > + // Verify log file is rotated by checking that > + // there are totally 4 log files on disk > + num_files = atoi(num_files_c); > + rc_validate(num_files, 2); > } > > -// Verify that log and cfg files are rotated when the number of log/cfg files > -// reach the "saLogStreamMaxFilesRotated" value (ticket #3045) > +// Verify that oldest log and cfg files are removed via admin operation with > +// admin id = 2 and parameter exist > // > -// Step1: Create and delete the app stream with > (saLogStreamMaxFilesRotated = 2) > -// 4 times > -// Step2: Verify that total is 4 cfg and log files (2 log files and 2 cfg files) > -void verRotatedLogCfgFile(void) > +// Step1: Create and delete the app stream with > (saLogStreamMaxFilesRotated = 4) > +// (There are two log files exists during create stream 2 times) > +// step2: Do admin operation to remove 1 oldest log files > +// step3: Verify 1 oldest log file is removed by checking that > +// there are 1 log files on disk > +void verRotatedLogCfgFile3(void) > { > - char command1[MAX_DATA], command2[MAX_DATA]; > - const char *object_dn = > - "safLgStrCfg=verRotatedFile,safApp=safLogService"; > - const uint32_t max_file = 2; > - char num_files_c[10]; > - uint32_t num_files = 0; > - > - // Command to create configuration application stream > - sprintf(command1, "immcfg -c SaLogStreamConfig %s -a > saLogStreamPathName=. " > - "-a saLogStreamFileName=verRotatedFile -a > saLogStreamMaxFilesRotated=%d", > - object_dn, max_file); > - // Command to delete configuration application stream > - sprintf(command2, "immcfg -d %s", object_dn); > - > - // Create and delete the app stream 4 times. > - // Log/cfg are created during creating stream. > - for (int i = 0; i < max_file + 1; ++i) { > - int rc = systemCall(command1); > - if (rc == 0) { > - osaf_nanosleep(&kHundredMilliseconds); > - rc = systemCall(command2); > - osaf_nanosleep(&kOneSecond); > - } > - if (rc != 0) { > - rc_validate(rc, 0); > - return; > - } > - } > - // Command to count number of log/cfg files on disk > - sprintf(command1,"find %s -type f -mmin -1 " > - "| egrep '%s(_.*)\\.[log$\\|cfg$]' " > - "| wc -l | awk '{printf $1}'", > - log_root_path, "verRotatedFile"); > - > - FILE *fp = popen(command1, "r"); > - > - // Get number of cfg and log file > - while (fgets(num_files_c, sizeof(num_files_c) - 1, fp) != NULL) {}; > - pclose(fp); > - > - // Verify cfg/log files are rotated by checking > - // that total is 4 cfg and log files > - num_files = atoi(num_files_c); > - rc_validate(num_files, max_file * 2); > + int rc; > + char command1[MAX_DATA], command2[MAX_DATA]; > + const char *object_dn = > + "safLgStrCfg=verRotatedFile3,safApp=safLogService"; > + char num_files_c[10]; > + uint32_t num_files = 0; > + > + // Command to create configuration application stream > + sprintf(command1, "immcfg -c SaLogStreamConfig %s " > + "-a saLogStreamPathName=. " > + "-a saLogStreamFileName=verRotatedFile3" > + " -a saLogStreamMaxFilesRotated=4", > + object_dn); > + // Command to delete configuration application stream > + sprintf(command2, "immcfg -d %s", object_dn); > + > + // Create and delete the app stream. > + // 2 log are created during creating stream. > + for (int i = 0; i < 2; ++i) { > + rc = systemCall(command1); > + if (rc == 0) { > + osaf_nanosleep(&kHundredMilliseconds); > + if (i == 1) break; // Keep opened stream at last turn > + rc = systemCall(command2); > + osaf_nanosleep(&kOneSecond); > + } > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + } > + > + // Admin operation to remove 1 oldest log files > + sprintf(command1, "immadm -o 2 -p > numberOfFilesToRemove:SA_UINT32_T:1 " > + "%s", object_dn); > + rc = systemCall(command1); > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + > + // Command to count number of log/cfg files on disk > + sprintf(command1, "find %s -type f -mmin -1 " > + "| egrep \"%s_([0-9]{8}_[0-9]{6}\\.log$)\" " > + "| wc -l | awk '{printf $1}'", > + log_root_path, "verRotatedFile3"); > + FILE *fp = popen(command1, "r"); > + // Get number of cfg and log file > + while (fgets(num_files_c, sizeof(num_files_c) - 1, fp) != NULL) {}; > + pclose(fp); > + > + // Close the application stream > + rc = systemCall(command2); > + if (rc != 0) { > + rc_validate(rc, 0); > + return; > + } > + > + // Verify 1 oldest cfg/log files are removed > + // by checking that there is 1 log files on disk > + num_files = atoi(num_files_c); > + rc_validate(num_files, 1); > } > > __attribute__((constructor)) static void saOiOperations_constructor(void) > @@ -5969,9 +6075,13 @@ __attribute__((constructor)) static void > saOiOperations_constructor(void) > 6, verLogFileRotate, > "Verify that log file is not rotated if file size doesn't reach max file > size (ticket1439)"); > test_case_add(6, admin_rotate_log_file, > - "Verify admin operation rotate log file is accepted"); > - test_case_add(6, admin_rotate_log_file_with_param, > - "Verify admin operation rotate log file with parameter fails"); > - test_case_add(6, verRotatedLogCfgFile, > - "Verify that both log and cfg files are rotated"); > + "Verify admin operation rotate log file is accepted"); > + test_case_add(6, verRotatedLogCfgFile, > + "Verify that both log and cfg files are rotated"); > + test_case_add(6, verRotatedLogCfgFile2, > + "Verify log file is rotated via admin operation " > + "without parameter"); > + test_case_add(6, verRotatedLogCfgFile3, > + "Verify oldest log files are removed via admin operation " > + "with parameter"); > } > diff --git a/src/log/logd/lgs_filehdl.cc b/src/log/logd/lgs_filehdl.cc > index b1afce3d5..e683f8b68 100644 > --- a/src/log/logd/lgs_filehdl.cc > +++ b/src/log/logd/lgs_filehdl.cc > @@ -552,7 +552,7 @@ static int check_log_oldest(char *line, char > *fname_prefix, > *old_date = date; > *old_time = time; > return 1; > - } else if ((date == *old_date) && (time < *old_time)) { > + } else if ((date == *old_date) && (time <= *old_time)) { > *old_date = date; > *old_time = time; > return 1; > @@ -800,7 +800,7 @@ int get_number_of_cfg_files_hdl(void *indata, void > *outdata, > } > } > > - if (old_ind == -1) goto done_log_free; > + if (old_ind == -1) goto done_cfg_free; > > // If the oldest cfg file is older than the oldest log file, the output > // is returned with oldest cfg file. Otherwise the output is returned empty > diff --git a/src/log/logd/lgs_imm.cc b/src/log/logd/lgs_imm.cc > index 49bc686b6..ec36098c8 100644 > --- a/src/log/logd/lgs_imm.cc > +++ b/src/log/logd/lgs_imm.cc > @@ -84,9 +84,11 @@ static std::vector<std::string> file_path_list; > static void lgs_admin_change_filter( > SaImmOiHandleT immOiHandle, SaInvocationT invocation, > SaConstStringT objectName, const SaImmAdminOperationParamsT_2 > *parameter); > -static void lgs_admin_rotate_log_file(SaImmOiHandleT immOiHandle, > - SaInvocationT invocation, > - SaConstStringT objectName); > +static void lgs_admin_log_file( > + SaImmOiHandleT immOiHandle, > + SaInvocationT invocation, > + SaConstStringT objectName, > + const SaImmAdminOperationParamsT_2 *parameter); > > static void report_oi_error(SaImmOiHandleT immOiHandle, SaImmOiCcbIdT > ccbId, > const char *format, ...) > @@ -396,12 +398,7 @@ static void adminOperationCallback( > } else if (opId == SA_LOG_ADMIN_CHANGE_FILTER) { > lgs_admin_change_filter(immOiHandle, invocation, objName, param); > } else if (opId == SA_LOG_ADMIN_ROTATE_FILE) { > - if (param != nullptr) { > - report_om_error(immOiHandle, invocation, > - "Admin op rotate log file: parameters is not empty"); > - } else { > - lgs_admin_rotate_log_file(immOiHandle, invocation, objName); > - } > + lgs_admin_log_file(immOiHandle, invocation, objName, param); > } else { > report_om_error( immOiHandle, invocation, "Invalid operation ID"); > } > @@ -3402,15 +3399,68 @@ static void lgs_admin_change_filter( > } > > /** > - * Handle admin operation that rotates the log file > + * Get number of files to remove from parameter in the invoked admin > operation > + * > + * @param immOiHandle > + * @param invocation > + * @param parameter > + * @return -1 if invalid parameter > + * 0 if parameter is null > + * > 0 number of files to remove > + */ > +static int get_number_of_files_to_remove( > + SaImmOiHandleT immOiHandle, > + SaInvocationT invocation, > + log_stream_t *stream, > + const SaImmAdminOperationParamsT_2 *parameter) { > + TRACE_ENTER(); > + int result; [Vu] Unnecessary variable > + if (parameter == nullptr) { > + result = 0; [Vu] return 0; > + } else if (strcmp(parameter->paramName, "numberOfFilesToRemove") != > 0) { [Vu] Remove else > + report_om_error(immOiHandle, invocation, > + "Admin op remove oldest log files, invalid param name. " > + "Should be 'numberOfFilesToRemove'"); > + result = -1; [Vu] Return -1 > + } else if (parameter->paramType != SA_IMM_ATTR_SAUINT32T) { [Vu] Remove else > + report_om_error(immOiHandle, invocation, > + "Admin op remove oldest log files: invalid parameter type. " > + "Should be 'SA_UINT32_T'"); > + result = -1; [Vu] return -1 > + } else { [Vu] remove else > + SaUint32T number_files_to_remove = > + *(reinterpret_cast<SaUint32T *>(parameter->paramBuffer)); > + > + if (number_files_to_remove < 1 || > + number_files_to_remove >= stream->maxFilesRotated) { > + report_om_error(immOiHandle, invocation, > + "Admin op remove oldest log files: Out of range " > + "(min:1 - max:%d)", stream->maxFilesRotated - 1); > + result = -1; [Vu] return -1 > + } else { [Vu] Remove else > + result = number_files_to_remove; [Vu] return number_files_to_remove > + } > + } > + > + TRACE_LEAVE2("num: %d", result); > + return result; > +} > + > +/** > + * Handle admin operation > + * - If the parameter is omitted, log/cfg file is rotated > + * - If the parameter exists, remove the oldest log/cfg files > * > * @param immOiHandle > * @param invocation > * @param objectName > + * @param parameter > */ > -static void lgs_admin_rotate_log_file(SaImmOiHandleT immOiHandle, > - SaInvocationT invocation, > - SaConstStringT objectName) { > +static void lgs_admin_log_file( > + SaImmOiHandleT immOiHandle, > + SaInvocationT invocation, > + SaConstStringT objectName, > + const SaImmAdminOperationParamsT_2 *parameter) { > TRACE_ENTER(); > > osafassert(objectName != nullptr); > @@ -3421,16 +3471,26 @@ static void > lgs_admin_rotate_log_file(SaImmOiHandleT immOiHandle, > return; > } > > - // Rotate log file. > - // Send the result "FAILED_OPERATION" to admin if the rotation fails > + // Send the result "FAILED_OPERATION" to admin if the rotation/deletion > fails > SaAisErrorT result = SA_AIS_OK; > - int ret = log_rotation_act(stream); > - if (ret == 0) > - // Checkpoint the stream to standby to rotate the log file > - // in case the split file system > - ckpt_stream_config(stream); > - else > - result = SA_AIS_ERR_FAILED_OPERATION; > + > + int number_of_files_to_remove = > + get_number_of_files_to_remove(immOiHandle, invocation, stream, > parameter); > + if (number_of_files_to_remove == -1) { [Vu] Does it need to call ' immutil_saImmOiAdminOperationResult' before return? > + return; > + } else if (number_of_files_to_remove == 0) { [Vu] else here is not necessary if you has 'return' in above if. > + // Rotate log file. > + if (log_rotation_act(stream) == 0) > + // Checkpoint the stream to standby to rotate the log file > + // in case the split file system > + ckpt_stream_config(stream); > + else > + result = SA_AIS_ERR_FAILED_OPERATION; > + } else { > + // Remove the oldest log files > + if (remove_oldest_log_files(stream, number_of_files_to_remove) == > false) > + result = SA_AIS_ERR_FAILED_OPERATION; > + } > > result = immutil_saImmOiAdminOperationResult(immOiHandle, invocation, > result); > if (result == SA_AIS_ERR_BAD_HANDLE) { > diff --git a/src/log/logd/lgs_stream.cc b/src/log/logd/lgs_stream.cc > index 7709c3c81..f2f7df940 100644 > --- a/src/log/logd/lgs_stream.cc > +++ b/src/log/logd/lgs_stream.cc > @@ -269,35 +269,50 @@ static int delete_config_file(log_stream_t > *stream) { > } > > /** > - * Remove oldest log file until there are 'maxFilesRotated' - 1 files left > - * The oldest cfg files are also removed > + * Remove oldest log and cfg files on disk. > + * If the parameter @number_files_to_remove is zero, remove older log > and cfg > + * files until there are "maxFilesRotated" - 1 files left on disk. > * > * @param stream > + * @param number_files_to_remove > * @return true/false > */ > -static bool rotate_if_needed(log_stream_t *stream) { > +bool remove_oldest_log_files(log_stream_t *stream, > + int number_files_to_remove) { > char oldest_log_file[PATH_MAX]; > char oldest_cfg_file[PATH_MAX]; > - const int max_files_rotated = static_cast<int>(stream->maxFilesRotated); > + int max_files_rotated = static_cast<int>(stream->maxFilesRotated) - 1; > > - TRACE_ENTER(); > + TRACE_ENTER2("num: %d", number_files_to_remove); > > // Get number of log files and the oldest log file > int log_file_cnt = get_number_of_log_files_h(stream, oldest_log_file); > - while (log_file_cnt >= max_files_rotated) { > + if (log_file_cnt == -1) return false; > + > + if (number_files_to_remove != 0) { > + // If there are not enough number of log file to remove, return true > + if (log_file_cnt <= number_files_to_remove) { > + LOG_WA("No (%d) older log files to remove on disk", > + number_files_to_remove); > + return true; > + } > + max_files_rotated = log_file_cnt - number_files_to_remove; > + } > + > + while (log_file_cnt > max_files_rotated) { > TRACE("Delete oldest_log_file %s", oldest_log_file); > if (file_unlink_h(oldest_log_file) == -1) { > LOG_NO("Delete log file fail: %s - %s", oldest_log_file, strerror(errno)); > return false; > } > log_file_cnt = get_number_of_log_files_h(stream, oldest_log_file); > + if (log_file_cnt == -1) return false; > } > - if (log_file_cnt == -1) return false; > > // Housekeeping for cfg files > int number_deleted_files = 0; > int cfg_file_cnt = get_number_of_cfg_files_h(stream, oldest_cfg_file); > - while (cfg_file_cnt >= max_files_rotated) { > + while (cfg_file_cnt > max_files_rotated) { > TRACE("Delete oldest_cfg_file %s", oldest_cfg_file); > if (file_unlink_h(oldest_cfg_file) == -1) { > LOG_NO("Delete cfg file fail: %s - %s", oldest_cfg_file, strerror(errno)); > @@ -351,8 +366,8 @@ void log_initiate_stream_files(log_stream_t *stream) > { > (void)delete_config_file(stream); > > /* Remove files from a previous life if needed */ > - if (rotate_if_needed(stream) == false) { > - TRACE("%s - rotate_if_needed() FAIL", __FUNCTION__); > + if (remove_oldest_log_files(stream) == false) { > + TRACE("%s - remove_oldest_log_files() FAIL", __FUNCTION__); > goto done; > } > > @@ -1100,7 +1115,7 @@ int log_rotation_stb(log_stream_t *stream) { > } > > // Remove oldest file if needed > - if (rotate_if_needed(stream) == false) { > + if (remove_oldest_log_files(stream) == false) { > TRACE("Old file removal failed"); > } > // Save new name for current log file and open it > @@ -1169,7 +1184,7 @@ int log_rotation_act(log_stream_t *stream) { > // Reset file size for current log file > stream->curFileSize = 0; > // Remove oldest file if needed > - if (rotate_if_needed(stream) == false) > + if (remove_oldest_log_files(stream) == false) > TRACE("Old file removal failed"); > > // Create a new file name that includes "open time stamp" and open the file > @@ -1523,7 +1538,7 @@ int log_stream_config_change(bool create_files_f, > const std::string &root_path, > } > } > > - rotate_if_needed(stream); > + remove_oldest_log_files(stream); > > /* Reset file size for new log file */ > stream->curFileSize = 0; > diff --git a/src/log/logd/lgs_stream.h b/src/log/logd/lgs_stream.h > index 2aa6922f4..8fdb5693d 100644 > --- a/src/log/logd/lgs_stream.h > +++ b/src/log/logd/lgs_stream.h > @@ -149,5 +149,6 @@ void lgs_ckpt_stream_open(log_stream_t *stream, > uint32_t client_id); > > int log_rotation_act(log_stream_t *stream); > int log_rotation_stb(log_stream_t *stream); > - > +bool remove_oldest_log_files(log_stream_t *stream, > + int number_files_to_remove = 0); > #endif // LOG_LOGD_LGS_STREAM_H_ > -- > 2.15.1 _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel