Shenfeng1011 opened a new issue #7755:
URL: https://github.com/apache/pulsar/issues/7755
version pulsar:2.5.2 hadoop:3.3.0
### 1. ./bin/pulsar-admin topics stats-internal nissan/original/gb32960
"ledgers" : [ {
"ledgerId" : 28,
"entries" : 8,
"size" : 3048,
"offloaded" : false
}, {
"ledgerId" : 295,
"entries" : 14,
"size" : 5348,
"offloaded" : false
}, {
"ledgerId" : 800,
"entries" : 0,
"size" : 0,
"offloaded" : false
} ]
### 2. ./bin/pulsar-admin topics offload --size-threshold 1K
nissan/original/gb32960
### 3../bin/pulsar-admin topics stats-internal nissan/original/gb32960
"ledgers" : [ {
"ledgerId" : 28,
"entries" : 8,
"size" : 3048,
"offloaded" : true
}, {
"ledgerId" : 295,
"entries" : 14,
"size" : 5348,
"offloaded" : true
}, {
"ledgerId" : 800,
"entries" : 0,
"size" : 0,
"offloaded" : false
} ]
### 4. /usr/local/Cellar/hadoop-3.3.0/bin/hdfs dfs -ls
/pulsar2/nissan/original/persistent/gb32960
drwxr-xr-x - shenfeng supergroup 0 2020-08-05 14:54
/pulsar2/nissan/original/persistent/gb32960/28-27f328da-b7ee-4eba-8a17-7463df0230b4
drwxr-xr-x - shenfeng supergroup 0 2020-08-05 14:54
/pulsar2/nissan/original/persistent/gb32960/295-1690feab-beb1-4df4-be57-f5b8d8960914
this means that data has offloaded successfully
but when query by presto
### 5. presto> select count(*) from pulsar."nissan/original"."gb32960";
get error
org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader
Failed to open FileStoreBackedReadHandleImpl: ManagerLedgerName:
nissan/original/persistent/gb32960, LegerId: 295, UUID:
1690feab-beb1-4df4-be57-f5b8d8960914
java.io.FileNotFoundException: File
nissan/original/persistent/gb32960/295-1690feab-beb1-4df4-be57-f5b8d8960914/data
does not exist
at
org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:666)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:987)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:656)
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:454)
at
org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1863)
at
org.apache.hadoop.io.MapFile$Reader.createDataFileReader(MapFile.java:460)
at org.apache.hadoop.io.MapFile$Reader.open(MapFile.java:433)
at org.apache.hadoop.io.MapFile$Reader.<init>(MapFile.java:403)
at
org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader.lambda$readOffloaded$0(FileSystemManagedLedgerOffloader.java:308)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
conf/filesystem_offload_core_site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://127.0.0.1:9001</value>
</property>
<property>
<name>basedir</name>
<value>/pulsar1</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/pulsar2</value>
</property>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
