Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapController.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapController.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapController.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapController.java Fri Apr 15 07:27:56 2005 @@ -70,7 +70,7 @@ /************************************************************************** * Protected concrete impl of abstract methods of - * GenericCongloemrateController class: + * GenericConglomerateController class: ************************************************************************** */ protected final void getRowPositionFromRowLocation( @@ -105,6 +105,93 @@ * Private/Protected methods of This class: ************************************************************************** */ + + /** + * Check and purge committed deleted rows on a page. + * <p> + * + * @return true, if no purging has been done on page, and thus latch + * can be released before end transaction. Otherwise the latch + * on the page can not be released before commit. + * + * @param page A non-null, latched page must be passed in. If all + * rows on page are purged, then page will be removed and + * latch released. + * + * @exception StandardException Standard exception policy. + **/ + protected final boolean purgeCommittedDeletes( + Page page) + throws StandardException + { + boolean purgingDone = false; + + // The number records that can be reclaimed is: + // total recs - recs_not_deleted + int num_possible_commit_delete = + page.recordCount() - page.nonDeletedRecordCount(); + + if (num_possible_commit_delete > 0) + { + // loop backward so that purges which affect the slot table + // don't affect the loop (ie. they only move records we + // have already looked at). + for (int slot_no = page.recordCount() - 1; + slot_no >= 0; + slot_no--) + { + boolean row_is_committed_delete = + page.isDeletedAtSlot(slot_no); + + if (row_is_committed_delete) + { + // At this point we only know that the row is + // deleted, not whether it is committed. + + // see if we can purge the row, by getting an + // exclusive lock on the row. If it is marked + // deleted and we can get this lock, then it + // must be a committed delete and we can purge + // it. + + RecordHandle rh = + page.fetchFromSlot( + (RecordHandle) null, + slot_no, + RowUtil.EMPTY_ROW, + RowUtil.EMPTY_ROW_FETCH_DESCRIPTOR, + true); + + row_is_committed_delete = + this.lockRowAtSlotNoWaitExclusive(rh); + + if (row_is_committed_delete) + { + purgingDone = true; + + page.purgeAtSlot(slot_no, 1, false); + } + } + } + } + if (page.recordCount() == 0) + { + + // Deallocate the current page with 0 rows on it. + this.removePage(page); + + // removePage guarantees to unlatch the page even if an + // exception is thrown. The page is protected against reuse + // because removePage locks it with a dealloc lock, so it + // is OK to release the latch even after a purgeAtSlot is + // called. + // @see ContainerHandle#removePage + + purgingDone = true; + } + + return(purgingDone); + } /** * Insert a new row into the heap.
Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapScan.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapScan.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapScan.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapScan.java Fri Apr 15 07:27:56 2005 @@ -147,6 +147,28 @@ } } + protected void setRowLocationArray( + RowLocation[] rowloc_array, + int index, + RecordHandle rh) + throws StandardException + { + if (rowloc_array[index] == null) + { + rowloc_array[index] = new HeapRowLocation(rh); + } + else + { + if (SanityManager.DEBUG) + { + SanityManager.ASSERT( + rowloc_array[index] instanceof HeapRowLocation); + } + + ((HeapRowLocation)rowloc_array[index]).setFrom(rh); + } + } + /** Fetch the row at the next position of the Scan. @@ -252,6 +274,16 @@ (BackingStoreHashtable) null, row_array.length, (int[]) null)); + } + + public int fetchNextGroup( + DataValueDescriptor[][] row_array, + RowLocation[] old_rowloc_array, + RowLocation[] new_rowloc_array) + throws StandardException + { + throw(StandardException.newException( + SQLState.HEAP_UNIMPLEMENTED_FEATURE)); } Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/sort/Scan.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/sort/Scan.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/sort/Scan.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/sort/Scan.java Fri Apr 15 07:27:56 2005 @@ -98,6 +98,17 @@ SQLState.SORT_IMPROPER_SCAN_METHOD); } + public int fetchNextGroup( + DataValueDescriptor[][] row_array, + RowLocation[] old_rowloc_array, + RowLocation[] new_rowloc_array) + throws StandardException + { + throw StandardException.newException( + SQLState.SORT_IMPROPER_SCAN_METHOD); + } + + /** * Insert all rows that qualify for the current scan into the input * Hash table. Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java Fri Apr 15 07:27:56 2005 @@ -423,6 +423,34 @@ setExtentFreePageStatus(true); } + protected long compressPages() + { + int compress_bitnum = -1; + + for (int i = extentLength - 1; i >= 0; i--) + { + if (freePages.isSet(i)) + { + freePages.clear(i); + compress_bitnum = i; + } + else + { + break; + } + } + + if (compress_bitnum >= 0) + { + extentLength = compress_bitnum; + return(extentStart + extentLength - 1); + } + else + { + return(-1); + } + + } protected long getExtentEnd() { @@ -764,14 +792,20 @@ for (int j = 0; j < 8; j++) { if (((1 << j) & free[i]) != 0) + { allocatedPageCount--; + } } } } if (SanityManager.DEBUG) - SanityManager.ASSERT(allocatedPageCount >= 0, - "number of allocated page < 0"); + { + SanityManager.ASSERT( + allocatedPageCount >= 0, + "number of allocated page < 0, val =" + allocatedPageCount + + "\nextent = " + toDebugString()); + } return allocatedPageCount; } Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocPage.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocPage.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocPage.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocPage.java Fri Apr 15 07:27:56 2005 @@ -958,7 +958,36 @@ } + /** + compress + @param myContainer the container object + */ + protected boolean compress( + FileContainer myContainer) + throws StandardException + { + boolean all_pages_compressed = false; + + if (SanityManager.DEBUG) + SanityManager.ASSERT(isLatched(), "page is not latched"); + + long last_valid_page = extent.compressPages(); + if (last_valid_page >= 0) + { + // a non-negative return means that pages can be returned to + // the operating system. + myContainer.truncatePages(last_valid_page); + + if (last_valid_page == this.getPageNumber()) + { + // all pages of the extent have been returned to OS. + all_pages_compressed = true; + } + } + + return(all_pages_compressed); + } /********************************************************************* * Extent Testing @@ -968,6 +997,4 @@ * *********************************************************************/ public static final String TEST_MULTIPLE_ALLOC_PAGE = SanityManager.DEBUG ? "TEST_MULTI_ALLOC_PAGE" : null; - } - Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainer.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainer.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainer.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainer.java Fri Apr 15 07:27:56 2005 @@ -165,6 +165,68 @@ /** + Release free space to the OS. + <P> + As is possible release any free space to the operating system. This + will usually mean releasing any free pages located at the end of the + file using the java truncate() interface. + + @exception StandardException Standard Cloudscape error policy + */ + public void compressContainer(BaseContainerHandle handle) + throws StandardException + { + RawTransaction ntt = handle.getTransaction().startNestedTopTransaction(); + + int mode = handle.getMode(); + + if (SanityManager.DEBUG) + { + SanityManager.ASSERT((mode & ContainerHandle.MODE_FORUPDATE) == + ContainerHandle.MODE_FORUPDATE, + "addPage handle not for update"); + } + + // if we are not in the same transaction as the one which created the + // container and the container may have logged some operation already, + // then we need to log allocation regardless of whether user changes + // are logged. Otherwise, the database will be corrupted if it + // crashed. + if ((mode & ContainerHandle.MODE_CREATE_UNLOGGED) == 0 && + (mode & ContainerHandle.MODE_UNLOGGED) == + ContainerHandle.MODE_UNLOGGED) + mode &= ~ContainerHandle.MODE_UNLOGGED; + + // make a handle which is tied to the ntt, not to the user transaction + // this handle is tied to. The container is already locked by the + // user transaction, open it nolock + BaseContainerHandle allocHandle = (BaseContainerHandle) + ntt.openContainer(identity, (LockingPolicy)null, mode); + + if (allocHandle == null) + { + throw StandardException.newException( + SQLState.DATA_ALLOC_NTT_CANT_OPEN, + new Long(getSegmentId()), + new Long(getContainerId())); + } + + // Latch this container, the commit will release the latch + ntt.getLockFactory().lockObject( + ntt, ntt, this, null, C_LockFactory.WAIT_FOREVER); + + try + { + compressContainer(ntt, allocHandle); + } + finally + { + ntt.commitNoSync(Transaction.RELEASE_LOCKS); + ntt.close(); + } + } + + /** Add a page to this container. <BR> MT - thread aware - @@ -650,6 +712,14 @@ int flag) throws StandardException; + protected abstract BasePage getPageForCompress( + BaseContainerHandle handle, + int flag, + long pageno) + throws StandardException; + + protected abstract void truncatePages(long lastValidPagenum); + /** Create a new page in the container. @@ -660,6 +730,11 @@ RawTransaction t, BaseContainerHandle allocHandle, boolean isOverflow) throws StandardException; + + protected abstract void compressContainer( + RawTransaction t, + BaseContainerHandle allocHandle) + throws StandardException; /** Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainerHandle.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainerHandle.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainerHandle.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BaseContainerHandle.java Fri Apr 15 07:27:56 2005 @@ -181,6 +181,22 @@ } /** + Release free space to the OS. + <P> + As is possible release any free space to the operating system. This + will usually mean releasing any free pages located at the end of the + file using the java truncate() interface. + + @exception StandardException Standard Cloudscape error policy + */ + public void compressContainer() throws StandardException + { + checkUpdateOpen(); + + container.compressContainer(this); + } + + /** Add a page to the container, if flag == ContainerHandle.ADD_PAGE_BULK, tell the container about it. @@ -336,6 +352,14 @@ checkUpdateOpen(); return container.getPageForInsert(this, flag); + } + + public Page getPageForCompress(int flag, long pageno) + throws StandardException + { + checkUpdateOpen(); + + return container.getPageForCompress(this, flag, pageno); } /** Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java Fri Apr 15 07:27:56 2005 @@ -412,10 +412,10 @@ */ public RecordHandle fetch( - RecordHandle handle, - Object[] row, - FormatableBitSet validColumns, - boolean forUpdate) + RecordHandle handle, + Object[] row, + FormatableBitSet validColumns, + boolean forUpdate) throws StandardException { if (SanityManager.DEBUG) { @@ -450,7 +450,7 @@ public RecordHandle fetchFromSlot( RecordHandle rh, int slot, - Object[] row, + Object[] row, FetchDescriptor fetchDesc, boolean ignoreDelete) throws StandardException @@ -1395,12 +1395,11 @@ // page does not copy over the remaining pieces, i.e.,the new head page // still points to those pieces. - owner.getActionSet().actionPurge (t, this, src_slot, num_rows, - recordIds, true); + owner.getActionSet().actionPurge( + t, this, src_slot, num_rows, recordIds, true); } - /** Unlatch the page. @see Page#unlatch @@ -2488,7 +2487,6 @@ */ public abstract boolean spaceForCopy(int num_rows, int[] spaceNeeded) throws StandardException; - /** Return the total number of bytes used, reserved, or wasted by the Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Fri Apr 15 07:27:56 2005 @@ -1281,6 +1281,124 @@ } + /** + Compress free space from container. + + <BR> MT - thread aware - It is assumed that our caller (our super class) + has already arranged a logical lock on page allocation to only allow a + single thread through here. + + Compressing free space is done in allocation page units, working + it's way from the end of the container to the beginning. Each + loop operates on the last allocation page in the container. + + Freeing space in the container page involves 2 transactions, an + update to an allocation page, N data pages, and possibly the delete + of the allocation page. + The User Transaction (UT) initiated the compress call. + The Nested Top Transaction (NTT) is the transaction started by RawStore + inside the compress call. This NTT is committed before compress returns. + The NTT is used to access high traffic data structures such as the + AllocPage. + + This is outline of the algorithm used in compressing the container. + + Until a non free page is found loop, in each loop return to the OS + all space at the end of the container occupied by free pages, including + the allocation page itself if all of it's pages are free. + + 1) Find last 2 allocation pages in container (last if there is only one). + 2) invalidate the allocation information cached by the container. + Without the cache no page can be gotten from the container. Pages + already in the page cache are not affected. Thus by latching the + allocPage and invalidating the allocation cache, this NTT blocks out + all page gets from this container until it commits. + 3) the allocPage determines which pages can be released to the OS, + mark that in its data structure (the alloc extent). Mark the + contiguous block of nallocated/free pages at the end of the file + as unallocated. This change is associated with the NTT. + 4) The NTT calls the OS to deallocate the space from the file. Note + that the system can handle being booted and asked to get an allocated + page which is past end of file, it just extends the file automatically. + 5) If freeing all space on the alloc page, and there is more than one + alloc page, then free the alloc page - this requires an update to the + previous alloc page which the loop has kept latched also. + 6) if the last alloc page was deleted, restart loop at #1 + + All NTT latches are released before this routine returns. + If we use an NTT, the caller has to commit the NTT to release the + allocPage latch. If we don't use an NTT, the allocPage latch is released + as this routine returns. + + @param ntt - the nested top transaction for the purpose of freeing space. + If ntt is null, use the user transaction for allocation. + #param allocHandle - the container handle opened by the ntt, + use this to latch the alloc page + + @exception StandardException Standard Cloudscape error policy + */ + protected void compressContainer( + RawTransaction ntt, + BaseContainerHandle allocHandle) + throws StandardException + { + AllocPage alloc_page = null; + AllocPage prev_alloc_page = null; + + if (firstAllocPageNumber == ContainerHandle.INVALID_PAGE_NUMBER) + { + // no allocation pages in container, no work to do! + return; + } + + try + { + synchronized(allocCache) + { + // loop until last 2 alloc pages are reached. + alloc_page = (AllocPage) + allocHandle.getAllocPage(firstAllocPageNumber); + + while (!alloc_page.isLast()) + { + if (prev_alloc_page != null) + { + // there are more than 2 alloc pages, unlatch the + // earliest one. + prev_alloc_page.unlatch(); + } + prev_alloc_page = alloc_page; + alloc_page = null; + + long nextAllocPageNumber = + prev_alloc_page.getNextAllocPageNumber(); + long nextAllocPageOffset = + prev_alloc_page.getNextAllocPageOffset(); + + alloc_page = (AllocPage) + allocHandle.getAllocPage(nextAllocPageNumber); + } + + alloc_page.compress(this); + + allocCache.invalidate(); + } + } + catch (StandardException se) + { + + if (alloc_page != null) + { + alloc_page.unlatch(); + alloc_page = null; + } + if (prev_alloc_page != null) + { + prev_alloc_page.unlatch(); + prev_alloc_page = null; + } + } + } /** Create a new page in the container. @@ -2487,6 +2605,15 @@ */ return p; } + + protected BasePage getPageForCompress( + BaseContainerHandle handle, + int flag, + long pageno) + throws StandardException + { + return(getPageForInsert(handle, flag)); + } /** Get a potentially suitable page for insert and latch it. Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java Fri Apr 15 07:27:56 2005 @@ -132,6 +132,13 @@ return 0; } + protected void truncatePages(long lastValidPagenum) + { + // Nothing to do since we are inherently read-only. + return; + } + + /* ** Container creation, opening, and closing */ Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Fri Apr 15 07:27:56 2005 @@ -537,6 +537,54 @@ return n; } + /** + * Short one line description of routine. + * <p> + * Longer descrption of routine. + * <p> + * + * @return The identifier to be used to open the conglomerate later. + * + * @param param1 param1 does this. + * @param param2 param2 does this. + * + * @exception StandardException Standard exception policy. + **/ + protected void truncatePages( + long lastValidPagenum) + { + // int n = doTruncatePages(lastValidPagenum); + + synchronized(this) + { + boolean inwrite = false; + try + { + dataFactory.writeInProgress(); + inwrite = true; + + fileData.setLength((lastValidPagenum + 1) * pageSize); + } + catch (IOException ioe) + { + // The disk may have run out of space. + // Don't error out in un-allocation since application can + // still function even if allocation fails. + } + catch (StandardException se) + { + // some problem calling writeInProgress + } + finally + { + if (inwrite) + dataFactory.writeFinished(); + } + } + + return; + } + /* Write the header of a random access file and sync it Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/StoredPage.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/StoredPage.java?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/StoredPage.java (original) +++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/StoredPage.java Fri Apr 15 07:27:56 2005 @@ -1374,6 +1374,17 @@ return((freeSpace - bytesNeeded) >= 0); } + protected boolean spaceForCopy(int spaceNeeded) + { + // add up the space needed by the rows, add in minimumRecordSize + // if length of actual row is less than minimumRecordSize. + int bytesNeeded = slotEntrySize + + (spaceNeeded >= minimumRecordSize ? + spaceNeeded : minimumRecordSize); + + return((freeSpace - bytesNeeded) >= 0); + } + /** * Read the record at the given slot into the given row. * <P> @@ -6807,6 +6818,149 @@ overflowPage.unlatch(); return count; } + + /** + * Move record to a page toward the beginning of the file. + * <p> + * As part of compressing the table records need to be moved from the + * end of the file toward the beginning of the file. Only the + * contiguous set of free pages at the very end of the file can + * be given back to the OS. This call is used to purge the row from + * the current page, insert it into a previous page, and return the + * new row location + * Mark the record identified by position as deleted. The record may be + * undeleted sometime later using undelete() by any transaction that sees + * the record. + * <p> + * The interface is optimized to work on a number of rows at a time, + * optimally processing all rows on the page at once. The call will + * process either all rows on the page, or the number of slots in the + * input arrays - whichever is smaller. + * <B>Locking Policy</B> + * <P> + * MUST be called with table locked, not locks are requested. Because + * it is called with table locks the call will go ahead and purge any + * row which is marked deleted. It will also use purge rather than + * delete to remove the old row after it moves it to a new page. This + * is ok since the table lock insures that no other transaction will + * use space on the table before this transaction commits. + * + * <BR> + * A page latch on the new page will be requested and released. + * + * @param old_handle An array to be filled in by the call with the + * old handles of all rows moved. + * @param new_handle An array to be filled in by the call with the + * new handles of all rows moved. + * @param new_pageno An array to be filled in by the call with the + * new page number of all rows moved. + * + * @return the number of rows processed. + * + * @exception StandardException Standard Cloudscape error policy + * + * @see LockingPolicy + **/ + public int moveRecordForCompressAtSlot( + int slot, + Object[] row, + RecordHandle[] old_handle, + RecordHandle[] new_handle) + throws StandardException + { + long src_pageno = getPageNumber(); + + try + { + fetchFromSlot( + null, + slot, + row, + (FetchDescriptor) null, // all columns retrieved + false); + + int row_size = getRecordPortionLength(slot); + + // first see if row will fit on current page being used to insert + StoredPage dest_page = + (StoredPage) owner.getPageForCompress(0, src_pageno); + + if (dest_page != null) + { + SanityManager.DEBUG_PRINT("moveRecordForCompressAtSlot", + "last = " + dest_page.getPageNumber()); + + if ((dest_page.getPageNumber() >= getPageNumber()) || + (!dest_page.spaceForCopy(row_size))) + { + // page won't work + dest_page.unlatch(); + dest_page = null; + } + } + + if (dest_page == null) + { + // last page did not work, try unfilled page + dest_page = (StoredPage) + owner.getPageForCompress( + ContainerHandle.GET_PAGE_UNFILLED, src_pageno); + + if (dest_page != null) + { + SanityManager.DEBUG_PRINT("moveRecordForCompressAtSlot", + "unfill = " + dest_page.getPageNumber()); + + if ((dest_page.getPageNumber() >= getPageNumber()) || + (!dest_page.spaceForCopy(row_size))) + { + // page won't work + dest_page.unlatch(); + dest_page = null; + } + } + } + + if (dest_page == null) + { + // last and unfilled page did not work, try getting a free page + dest_page = (StoredPage) owner.addPage(); + + SanityManager.DEBUG_PRINT("moveRecordForCompressAtSlot", + "addPage = " + dest_page.getPageNumber()); + + if (dest_page.getPageNumber() >= getPageNumber()) + { + owner.removePage(dest_page); + dest_page = null; + } + } + + if (dest_page != null) + { + int dest_slot = dest_page.recordCount(); + + old_handle[0] = getRecordHandleAtSlot(slot); + + copyAndPurge(dest_page, slot, 1, dest_slot); + + new_handle[0] = dest_page.getRecordHandleAtSlot(dest_slot); + + dest_page.unlatch(); + + return(1); + } + else + { + return(0); + } + } + catch (IOException ioe) + { + throw StandardException.newException( + SQLState.DATA_UNEXPECTED_EXCEPTION, ioe); + } + } /* * methods that is called underneath a page action Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/onlineCompressTable.out URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/onlineCompressTable.out?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/onlineCompressTable.out (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/onlineCompressTable.out Fri Apr 15 07:27:56 2005 @@ -0,0 +1,88 @@ +ij> autocommit off; +ij> -- start with simple test, does the call work? +create table test1 (a int); +0 rows inserted/updated/deleted +ij> -- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('APP', 'TEST1'); +-- expect failures schema/table does not exist +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE(null, 'test2'); +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('APP', 'test2'); +-- non existent schema +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('doesnotexist', 'a'); +-- cleanup +drop table test1; +0 rows inserted/updated/deleted +ij> -- load up a table, delete most of it's rows and then see what compress does. +create table test1 (keycol int, a char(250), b char(250), c char(250), d char(250)); +0 rows inserted/updated/deleted +ij> insert into test1 values (1, 'a', 'b', 'c', 'd'); +1 row inserted/updated/deleted +ij> insert into test1 (select keycol + 1, a, b, c, d from test1); +1 row inserted/updated/deleted +ij> insert into test1 (select keycol + 2, a, b, c, d from test1); +2 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 4, a, b, c, d from test1); +4 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 8, a, b, c, d from test1); +8 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 16, a, b, c, d from test1); +16 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 32, a, b, c, d from test1); +32 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 64, a, b, c, d from test1); +64 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 128, a, b, c, d from test1); +128 rows inserted/updated/deleted +ij> insert into test1 (select keycol + 256, a, b, c, d from test1); +256 rows inserted/updated/deleted +ij> create index test1_idx on test1(keycol); +0 rows inserted/updated/deleted +ij> commit; +ij> select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; +CONGLOMERATENAME |ISIND&|NUMALLOCATEDPAGES |NUMFREEPAGES |PAGESIZE |ESTIMSPACESAVING +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +TEST1 |0 |171 |0 |4096 |0 +TEST1_IDX |1 |4 |0 |4096 |0 +ij> delete from test1 where keycol > 300; +212 rows inserted/updated/deleted +ij> commit; +ij> delete from test1 where keycol < 100; +99 rows inserted/updated/deleted +ij> commit; +ij> call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 1, 0, 0); +0 rows inserted/updated/deleted +ij> select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; +CONGLOMERATENAME |ISIND&|NUMALLOCATEDPAGES |NUMFREEPAGES |PAGESIZE |ESTIMSPACESAVING +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +TEST1 |0 |68 |103 |4096 |421888 +TEST1_IDX |1 |4 |0 |4096 |0 +ij> commit; +ij> -- call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 0, 1, 0); +select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; +CONGLOMERATENAME |ISIND&|NUMALLOCATEDPAGES |NUMFREEPAGES |PAGESIZE |ESTIMSPACESAVING +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +TEST1 |0 |68 |103 |4096 |421888 +TEST1_IDX |1 |4 |0 |4096 |0 +ij> call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 0, 0, 1); +0 rows inserted/updated/deleted +ij> select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; +CONGLOMERATENAME |ISIND&|NUMALLOCATEDPAGES |NUMFREEPAGES |PAGESIZE |ESTIMSPACESAVING +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +TEST1 |0 |68 |32 |4096 |131072 +TEST1_IDX |1 |4 |0 |4096 |0 +ij> Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/storetests.runall URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/storetests.runall?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/storetests.runall (original) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/storetests.runall Fri Apr 15 07:27:56 2005 @@ -1,4 +1,5 @@ storetests/st_schema.sql +storetests/onlineCompressTable.sql storetests/st_1.sql storetests/st_b5772.sql storetests/derby94.sql Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/BaseTest.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/BaseTest.java?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/BaseTest.java (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/BaseTest.java Fri Apr 15 07:27:56 2005 @@ -0,0 +1,78 @@ +/* + + Derby - Class org.apache.derbyTesting.functionTests.harness.procedure + + Copyright 2005 The Apache Software Foundation or its licensors, as applicable. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package org.apache.derbyTesting.functionTests.tests.store; + +import org.apache.derby.tools.ij; + +import java.sql.Connection; +import java.sql.SQLException; + + +public abstract class BaseTest +{ + abstract void testList(Connection conn) throws SQLException; + + void runTests(String[] argv) + throws Throwable + { + ij.getPropertyArg(argv); + Connection conn = ij.startJBMS(); + System.out.println("conn from ij.startJBMS() = " + conn); + conn.setAutoCommit(false); + + try + { + testList(conn); + } + catch (SQLException sqle) + { + org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException( + System.out, sqle); + sqle.printStackTrace(System.out); + } + } + + public BaseTest() + { + } + + protected void beginTest( + Connection conn, + String str) + throws SQLException + { + log("Beginning test: " + str); + conn.commit(); + } + + protected void endTest( + Connection conn, + String str) + throws SQLException + { + conn.commit(); + log("Ending test: " + str); + } + protected void log(String str) + { + System.out.println(str); + } +} Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java Fri Apr 15 07:27:56 2005 @@ -0,0 +1,120 @@ +/* + + Derby - Class org.apache.derbyTesting.functionTests.harness.procedure + + Copyright 2005 The Apache Software Foundation or its licensors, as applicable. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +package org.apache.derbyTesting.functionTests.tests.store; + +import org.apache.derby.iapi.db.OnlineCompress; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.derby.tools.ij; + + +public class OnlineCompressTest extends BaseTest +{ + + OnlineCompressTest() + { + } + + + private void createAndLoadTable( + Connection conn, + String tblname) + throws SQLException + { + Statement s = conn.createStatement(); + + s.execute( + "create table " + tblname + + "(keycol int, indcol1 int, indcol2 int, indcol3 int, data1 varchar(2000), data2 varchar(2000))"); + + PreparedStatement insert_stmt = + conn.prepareStatement( + "insert into " + tblname + " values(?, ?, ?, ?, ?, ?)"); + + char[] data1_data = new char[500]; + char[] data2_data = new char[500]; + + for (int i = 0; i < data1_data.length; i++) + { + data1_data[i] = 'a'; + data2_data[i] = 'b'; + } + + String data1_str = new String(data1_data); + String data2_str = new String(data2_data); + + for (int i = 0; i < 10000; i++) + { + insert_stmt.setInt(1, i); // keycol + insert_stmt.setInt(2, i * 10); // indcol1 + insert_stmt.setInt(3, i * 100); // indcol2 + insert_stmt.setInt(4, -i); // indcol3 + insert_stmt.setString(5, data1_str); // data1_data + insert_stmt.setString(6, data2_str); // data2_data + } + + conn.commit(); + } + + private void test1(Connection conn) + throws SQLException + { + beginTest(conn, "test1"); + + createAndLoadTable(conn, "test1"); + + OnlineCompress.compressTable("APP", "TEST1", true, true, true); + + endTest(conn, "test1"); + } + + public void testList(Connection conn) + throws SQLException + { + test1(conn); + } + + public static void main(String[] argv) + throws Throwable + { + OnlineCompressTest test = new OnlineCompressTest(); + + ij.getPropertyArg(argv); + Connection conn = ij.startJBMS(); + System.out.println("conn 2 from ij.startJBMS() = " + conn); + conn.setAutoCommit(false); + + try + { + test.testList(conn); + } + catch (SQLException sqle) + { + org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException( + System.out, sqle); + sqle.printStackTrace(System.out); + } + } +} Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_app.properties URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_app.properties?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_app.properties (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_app.properties Fri Apr 15 07:27:56 2005 @@ -0,0 +1 @@ +usedefaults=true Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties Fri Apr 15 07:27:56 2005 @@ -0,0 +1 @@ +usedefaults=true Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant (original) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/copyfiles.ant Fri Apr 15 07:27:56 2005 @@ -123,3 +123,5 @@ xaOffline1_sed.properties xab2354.sql xab2354_sed.properties +OnlineCompressTest_app.properties +OnlineCompressTest_derby.properties Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/BaseTest.java URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/BaseTest.java?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/BaseTest.java (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/BaseTest.java Fri Apr 15 07:27:56 2005 @@ -0,0 +1,80 @@ +/* + + Derby - Class org.apache.derbyTesting.functionTests.harness.procedure + + Copyright 2005 The Apache Software Foundation or its licensors, as applicable. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package org.apache.derbyTesting.functionTests.tests.storetests; + +import com.ibm.db2j.functionTests.TestUtil; + +import org.apache.derby.tools.ij; + +import java.sql.Connection; +import java.sql.SQLException; + + +public abstract class BaseTest +{ + abstract void testList(Connection conn) throws SQLException; + + void runTests(String[] argv) + throws Throwable + { + ij.getPropertyArg(argv); + Connection conn = ij.startJBMS(); + System.out.println("conn from ij.startJBMS() = " + conn); + conn.setAutoCommit(false); + + try + { + testList(conn); + } + catch (SQLException sqle) + { + org.apache.derby.tools.JDBCDisplayUtil.ShowSQLException( + System.out, sqle); + sqle.printStackTrace(System.out); + } + } + + public BaseTest() + { + } + + protected void beginTest( + Connection conn, + String str) + throws SQLException + { + log("Beginning test: " + str); + conn.commit(); + } + + protected void endTest( + Connection conn, + String str) + throws SQLException + { + conn.commit(); + log("Ending test: " + str); + } + protected void log(String str) + { + System.out.println(str); + } +} Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/copyfiles.ant URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/copyfiles.ant?view=diff&r1=161467&r2=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/copyfiles.ant (original) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/copyfiles.ant Fri Apr 15 07:27:56 2005 @@ -6,3 +6,4 @@ st_b5772.sql derby94.sql derby94_derby.properties +onlineCompressTable.sql Added: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/onlineCompressTable.sql URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/onlineCompressTable.sql?view=auto&rev=161468 ============================================================================== --- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/onlineCompressTable.sql (added) +++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/onlineCompressTable.sql Fri Apr 15 07:27:56 2005 @@ -0,0 +1,68 @@ +autocommit off; +-- start with simple test, does the call work? +create table test1 (a int); +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('APP', 'TEST1'); + +-- expect failures schema/table does not exist +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE(null, 'test2'); +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('APP', 'test2'); + +-- non existent schema +-- call SYSCS_UTIL.SYSCS_ONLINE_COMPRESS_TABLE('doesnotexist', 'a'); + +-- cleanup +drop table test1; + + +-- load up a table, delete most of it's rows and then see what compress does. +create table test1 (keycol int, a char(250), b char(250), c char(250), d char(250)); +insert into test1 values (1, 'a', 'b', 'c', 'd'); +insert into test1 (select keycol + 1, a, b, c, d from test1); +insert into test1 (select keycol + 2, a, b, c, d from test1); +insert into test1 (select keycol + 4, a, b, c, d from test1); +insert into test1 (select keycol + 8, a, b, c, d from test1); +insert into test1 (select keycol + 16, a, b, c, d from test1); +insert into test1 (select keycol + 32, a, b, c, d from test1); +insert into test1 (select keycol + 64, a, b, c, d from test1); +insert into test1 (select keycol + 128, a, b, c, d from test1); +insert into test1 (select keycol + 256, a, b, c, d from test1); + +create index test1_idx on test1(keycol); +commit; + +select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; + +delete from test1 where keycol > 300; +commit; +delete from test1 where keycol < 100; +commit; + + +call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 1, 0, 0); + +select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; +commit; + +-- call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 0, 1, 0); + +select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename; + +call SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE('APP', 'TEST1', 0, 0, 1); + +select + conglomeratename, isindex, numallocatedpages, numfreepages, pagesize, + estimspacesaving + from new org.apache.derby.diag.SpaceTable('TEST1') t + order by conglomeratename;