2021-09-27T00:00:00,014 ERROR [Nuxeo-Work-default-184:2174073272412070.116385061] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T00:00:00,018 ERROR [Nuxeo-Work-default-184] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-184, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2174073272412070.116385061 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T00:07:00,462 ERROR [Nuxeo-Work-default-91:2174493271604650.1632750323] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19VKQKT6HPY1D78; S3 Extended Request ID: y4h21GUqHuPqgL6bYElLJhUkGFMd5taSTU+TR6XU5Bly0zDADv7qMyjQ/vOcV242QMD/SO75Q7A=; Proxy: null), S3 Extended Request ID: y4h21GUqHuPqgL6bYElLJhUkGFMd5taSTU+TR6XU5Bly0zDADv7qMyjQ/vOcV242QMD/SO75Q7A= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19VKQKT6HPY1D78; S3 Extended Request ID: y4h21GUqHuPqgL6bYElLJhUkGFMd5taSTU+TR6XU5Bly0zDADv7qMyjQ/vOcV242QMD/SO75Q7A=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T00:07:00,564 ERROR [Nuxeo-Work-default-91:2174493271604650.1632750323] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19YWAQEWF45803F; S3 Extended Request ID: 9Ho4RoIy685K22z0UxSbE9F2f7H8BydrfimKe/zjoX0spzTHP3PDDLFhYOoIoevTdS0+e2nmXTQ=; Proxy: null), S3 Extended Request ID: 9Ho4RoIy685K22z0UxSbE9F2f7H8BydrfimKe/zjoX0spzTHP3PDDLFhYOoIoevTdS0+e2nmXTQ= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19YWAQEWF45803F; S3 Extended Request ID: 9Ho4RoIy685K22z0UxSbE9F2f7H8BydrfimKe/zjoX0spzTHP3PDDLFhYOoIoevTdS0+e2nmXTQ=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T00:07:00,665 ERROR [Nuxeo-Work-default-91:2174493271604650.1632750323] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19V0J7779GAZJWX; S3 Extended Request ID: Mn8iGvs30LEhhNdR61EGwDaQOEzoxN/sGk7B3Hk35dSzNz7yjpQ2oM19Egc6GJvpNNlb0WNDK34=; Proxy: null), S3 Extended Request ID: Mn8iGvs30LEhhNdR61EGwDaQOEzoxN/sGk7B3Hk35dSzNz7yjpQ2oM19Egc6GJvpNNlb0WNDK34= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19V0J7779GAZJWX; S3 Extended Request ID: Mn8iGvs30LEhhNdR61EGwDaQOEzoxN/sGk7B3Hk35dSzNz7yjpQ2oM19Egc6GJvpNNlb0WNDK34=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T00:07:00,766 ERROR [Nuxeo-Work-default-91:2174493271604650.1632750323] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19VE0M7G5EYRQRZ; S3 Extended Request ID: E4I3YAO+E+19hq0/gzGdqyMRi6cg22k8XjwIWva4a9oQZxgpN/qq9SuJ+C8+QbEC3rmb8A/rDYk=; Proxy: null), S3 Extended Request ID: E4I3YAO+E+19hq0/gzGdqyMRi6cg22k8XjwIWva4a9oQZxgpN/qq9SuJ+C8+QbEC3rmb8A/rDYk= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: H19VE0M7G5EYRQRZ; S3 Extended Request ID: E4I3YAO+E+19hq0/gzGdqyMRi6cg22k8XjwIWva4a9oQZxgpN/qq9SuJ+C8+QbEC3rmb8A/rDYk=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T00:15:00,014 ERROR [Nuxeo-Work-default-187:2174973272088494.638662070] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T00:15:00,022 ERROR [Nuxeo-Work-default-187] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-187, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2174973272088494.638662070 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T00:30:00,018 ERROR [Nuxeo-Work-default-189:2175873275161158.1630940475] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T00:30:00,021 ERROR [Nuxeo-Work-default-189] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-189, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2175873275161158.1630940475 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T00:45:00,014 ERROR [Nuxeo-Work-default-190:2176773271853904.274216594] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T00:45:00,022 ERROR [Nuxeo-Work-default-190] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-190, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2176773271853904.274216594 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T01:00:00,016 ERROR [Nuxeo-Work-default-191:2177673273675132.1831454943] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T01:00:00,024 ERROR [Nuxeo-Work-default-191] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-191, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2177673273675132.1831454943 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T01:07:00,477 ERROR [Nuxeo-Work-default-92:2178093277822893.1977607263] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8NDECDMXEV1AR69; S3 Extended Request ID: R6/V4mHNt0gV+kFSc5q7sDFWkVghnr7tDAMBpYaWJes8nyTUgA2Eh7xqUz4u7FZ5iUhA+7Wa+Sg=; Proxy: null), S3 Extended Request ID: R6/V4mHNt0gV+kFSc5q7sDFWkVghnr7tDAMBpYaWJes8nyTUgA2Eh7xqUz4u7FZ5iUhA+7Wa+Sg= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8NDECDMXEV1AR69; S3 Extended Request ID: R6/V4mHNt0gV+kFSc5q7sDFWkVghnr7tDAMBpYaWJes8nyTUgA2Eh7xqUz4u7FZ5iUhA+7Wa+Sg=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T01:07:00,579 ERROR [Nuxeo-Work-default-92:2178093277822893.1977607263] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8NBPH8YYE86VBDP; S3 Extended Request ID: AJ44kMWtsowAKTEqVYgBXgqUS7MhQNLRz994d2ecP/FL3uA2N1rl8m69pRoCovhZ/aBTJel/5Ng=; Proxy: null), S3 Extended Request ID: AJ44kMWtsowAKTEqVYgBXgqUS7MhQNLRz994d2ecP/FL3uA2N1rl8m69pRoCovhZ/aBTJel/5Ng= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8NBPH8YYE86VBDP; S3 Extended Request ID: AJ44kMWtsowAKTEqVYgBXgqUS7MhQNLRz994d2ecP/FL3uA2N1rl8m69pRoCovhZ/aBTJel/5Ng=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T01:07:00,680 ERROR [Nuxeo-Work-default-92:2178093277822893.1977607263] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8N8MYJJ7Y1Y8G38; S3 Extended Request ID: fnJ9C3W8w43i3A9k1vZovIj7acCvPHFmHyGkbfMetoHphMS7wPtXV6jHgTe6nwx3y643Rc7qFP8=; Proxy: null), S3 Extended Request ID: fnJ9C3W8w43i3A9k1vZovIj7acCvPHFmHyGkbfMetoHphMS7wPtXV6jHgTe6nwx3y643Rc7qFP8= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8N8MYJJ7Y1Y8G38; S3 Extended Request ID: fnJ9C3W8w43i3A9k1vZovIj7acCvPHFmHyGkbfMetoHphMS7wPtXV6jHgTe6nwx3y643Rc7qFP8=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T01:07:00,782 ERROR [Nuxeo-Work-default-92:2178093277822893.1977607263] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8N2RCHCA1MH74T0; S3 Extended Request ID: vw7g4lQJbu8IwZ8SxziZaUuB1mc/qavj0TxVjcaN4M6J9mZLDmdRGx5I+fdc9eVs6P1dvJ+suck=; Proxy: null), S3 Extended Request ID: vw7g4lQJbu8IwZ8SxziZaUuB1mc/qavj0TxVjcaN4M6J9mZLDmdRGx5I+fdc9eVs6P1dvJ+suck= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: B8N2RCHCA1MH74T0; S3 Extended Request ID: vw7g4lQJbu8IwZ8SxziZaUuB1mc/qavj0TxVjcaN4M6J9mZLDmdRGx5I+fdc9eVs6P1dvJ+suck=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T01:15:00,013 ERROR [Nuxeo-Work-default-188:2178573271369225.1946227698] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T01:15:00,016 ERROR [Nuxeo-Work-default-188] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-188, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2178573271369225.1946227698 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T01:30:00,017 ERROR [Nuxeo-Work-default-193:2179473274910858.1056903085] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T01:30:00,026 ERROR [Nuxeo-Work-default-193] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-193, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2179473274910858.1056903085 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T01:45:00,015 ERROR [Nuxeo-Work-default-194:2180373272351926.439680013] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T01:45:00,017 ERROR [Nuxeo-Work-default-194] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-194, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2180373272351926.439680013 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T02:00:00,018 ERROR [Nuxeo-Work-default-195:2181273276180504.440652821] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T02:00:00,025 ERROR [Nuxeo-Work-default-195] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-195, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2181273276180504.440652821 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T02:07:00,497 ERROR [Nuxeo-Work-default-91:2181693272058167.1845303956] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49G21NE03V89H08V; S3 Extended Request ID: ZBGCSEWuLwbxYZdQlIB4SFY/Wh5A3WlA8z0d7vZAhKMMynt8JzrrUZkYm2VZo/J2ak+iEfnyqtU=; Proxy: null), S3 Extended Request ID: ZBGCSEWuLwbxYZdQlIB4SFY/Wh5A3WlA8z0d7vZAhKMMynt8JzrrUZkYm2VZo/J2ak+iEfnyqtU= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49G21NE03V89H08V; S3 Extended Request ID: ZBGCSEWuLwbxYZdQlIB4SFY/Wh5A3WlA8z0d7vZAhKMMynt8JzrrUZkYm2VZo/J2ak+iEfnyqtU=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T02:07:00,599 ERROR [Nuxeo-Work-default-91:2181693272058167.1845303956] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49GF0NN9M7KNRXGD; S3 Extended Request ID: ZAh5zjOdeW3eafsUcYFPplrXhhX2pRbHGgnegzwCj3y3F1anooCfm8P7dyvl+2vQ4BRdEIRT9tY=; Proxy: null), S3 Extended Request ID: ZAh5zjOdeW3eafsUcYFPplrXhhX2pRbHGgnegzwCj3y3F1anooCfm8P7dyvl+2vQ4BRdEIRT9tY= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49GF0NN9M7KNRXGD; S3 Extended Request ID: ZAh5zjOdeW3eafsUcYFPplrXhhX2pRbHGgnegzwCj3y3F1anooCfm8P7dyvl+2vQ4BRdEIRT9tY=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T02:07:00,701 ERROR [Nuxeo-Work-default-91:2181693272058167.1845303956] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49GB206PW3D9R38B; S3 Extended Request ID: kipLs2xTvsgBuqtsMN6m6ZZRRyQxSPhmO/OrzWFiKd9YyjE2sjPw4SCvkI55/9U0q1mP96WfO3o=; Proxy: null), S3 Extended Request ID: kipLs2xTvsgBuqtsMN6m6ZZRRyQxSPhmO/OrzWFiKd9YyjE2sjPw4SCvkI55/9U0q1mP96WfO3o= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49GB206PW3D9R38B; S3 Extended Request ID: kipLs2xTvsgBuqtsMN6m6ZZRRyQxSPhmO/OrzWFiKd9YyjE2sjPw4SCvkI55/9U0q1mP96WfO3o=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T02:07:00,803 ERROR [Nuxeo-Work-default-91:2181693272058167.1845303956] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49G07BKTH3N25GBG; S3 Extended Request ID: 42gESIXWazVX4ydUHjuWm1D2hLXjIUYOywuPXOsQI+rdDzIoojSAOoth0GQ4dw+e7t6Or2UgXxM=; Proxy: null), S3 Extended Request ID: 42gESIXWazVX4ydUHjuWm1D2hLXjIUYOywuPXOsQI+rdDzIoojSAOoth0GQ4dw+e7t6Or2UgXxM= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 49G07BKTH3N25GBG; S3 Extended Request ID: 42gESIXWazVX4ydUHjuWm1D2hLXjIUYOywuPXOsQI+rdDzIoojSAOoth0GQ4dw+e7t6Or2UgXxM=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T02:15:00,016 ERROR [Nuxeo-Work-default-192:2182173274881386.349910975] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T02:15:00,020 ERROR [Nuxeo-Work-default-192] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-192, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2182173274881386.349910975 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T02:30:00,014 ERROR [Nuxeo-Work-default-197:2183073272816876.1887611215] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T02:30:00,024 ERROR [Nuxeo-Work-default-197] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-197, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2183073272816876.1887611215 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T02:45:00,015 ERROR [Nuxeo-Work-default-198:2183973272220239.1640927004] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T02:45:00,017 ERROR [Nuxeo-Work-default-198] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-198, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2183973272220239.1640927004 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T03:00:00,016 ERROR [Nuxeo-Work-default-199:2184873273177358.811240501] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T03:00:00,023 ERROR [Nuxeo-Work-default-199] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-199, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2184873273177358.811240501 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T03:07:00,493 ERROR [Nuxeo-Work-default-92:2185293272282337.646703833] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR416CCP1H1ACFR3; S3 Extended Request ID: iiLJOGfZCYo8eSMosrEh6gERUfAkKm0BlpGX47EQpE5WMNacRtsnbKTznIS3i4rRkzMDFnNYUYQ=; Proxy: null), S3 Extended Request ID: iiLJOGfZCYo8eSMosrEh6gERUfAkKm0BlpGX47EQpE5WMNacRtsnbKTznIS3i4rRkzMDFnNYUYQ= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR416CCP1H1ACFR3; S3 Extended Request ID: iiLJOGfZCYo8eSMosrEh6gERUfAkKm0BlpGX47EQpE5WMNacRtsnbKTznIS3i4rRkzMDFnNYUYQ=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T03:07:00,596 ERROR [Nuxeo-Work-default-92:2185293272282337.646703833] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR48XYV5J0F7WHBR; S3 Extended Request ID: eqY4VyiIINfB1ust0ho8oE7x8InK10lf5/mlNZSA4eVF2i5Jai2gOQw3rq32F2c1HRmL5RG2Iys=; Proxy: null), S3 Extended Request ID: eqY4VyiIINfB1ust0ho8oE7x8InK10lf5/mlNZSA4eVF2i5Jai2gOQw3rq32F2c1HRmL5RG2Iys= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR48XYV5J0F7WHBR; S3 Extended Request ID: eqY4VyiIINfB1ust0ho8oE7x8InK10lf5/mlNZSA4eVF2i5Jai2gOQw3rq32F2c1HRmL5RG2Iys=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T03:07:00,700 ERROR [Nuxeo-Work-default-92:2185293272282337.646703833] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR4B0VZDM38FRHPV; S3 Extended Request ID: p2rV885cHBmw1m+kwZ4I2GYkJYkruOjLeLeX4a/q0mt+ADW55KLTxKiNDWon4uWUQ6HXl5eO7JA=; Proxy: null), S3 Extended Request ID: p2rV885cHBmw1m+kwZ4I2GYkJYkruOjLeLeX4a/q0mt+ADW55KLTxKiNDWon4uWUQ6HXl5eO7JA= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR4B0VZDM38FRHPV; S3 Extended Request ID: p2rV885cHBmw1m+kwZ4I2GYkJYkruOjLeLeX4a/q0mt+ADW55KLTxKiNDWon4uWUQ6HXl5eO7JA=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T03:07:00,803 ERROR [Nuxeo-Work-default-92:2185293272282337.646703833] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR4ASJ02TTSVGVJC; S3 Extended Request ID: ySVlZIyMdYShOz8lE8ZsrR6YBjkfZXbQYDzbg9dKH4Uumw1M6HwQ2eqTiJK6cjINca6Usc+eoa8=; Proxy: null), S3 Extended Request ID: ySVlZIyMdYShOz8lE8ZsrR6YBjkfZXbQYDzbg9dKH4Uumw1M6HwQ2eqTiJK6cjINca6Usc+eoa8= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: PR4ASJ02TTSVGVJC; S3 Extended Request ID: ySVlZIyMdYShOz8lE8ZsrR6YBjkfZXbQYDzbg9dKH4Uumw1M6HwQ2eqTiJK6cjINca6Usc+eoa8=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T03:15:00,015 ERROR [Nuxeo-Work-default-196:2185773273668318.1896339520] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T03:15:00,024 ERROR [Nuxeo-Work-default-196] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-196, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2185773273668318.1896339520 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T03:30:00,019 ERROR [Nuxeo-Work-default-201:2186673275841186.1709490843] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T03:30:00,022 ERROR [Nuxeo-Work-default-201] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-201, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2186673275841186.1709490843 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T03:45:00,020 ERROR [Nuxeo-Work-default-202:2187573276550282.583333348] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T03:45:00,030 ERROR [Nuxeo-Work-default-202] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-202, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2187573276550282.583333348 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T04:00:00,017 ERROR [Nuxeo-Work-default-203:2188473274391082.1279447023] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T04:00:00,020 ERROR [Nuxeo-Work-default-203] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-203, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2188473274391082.1279447023 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T04:07:00,453 ERROR [Nuxeo-Work-default-91:2188893273296613.294792706] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DVXV5X2QCVC546; S3 Extended Request ID: q5mkFTsGsXBCnYqXjRAijhFsge3DCIiw50QofiXddT1MdzjSjo2YrzS/GdK4A03sJPWlbdTvR8E=; Proxy: null), S3 Extended Request ID: q5mkFTsGsXBCnYqXjRAijhFsge3DCIiw50QofiXddT1MdzjSjo2YrzS/GdK4A03sJPWlbdTvR8E= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DVXV5X2QCVC546; S3 Extended Request ID: q5mkFTsGsXBCnYqXjRAijhFsge3DCIiw50QofiXddT1MdzjSjo2YrzS/GdK4A03sJPWlbdTvR8E=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T04:07:00,556 ERROR [Nuxeo-Work-default-91:2188893273296613.294792706] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DQSH5CZWAH2JR3; S3 Extended Request ID: /8BZO1eA4zZkeRJeVFnDlljCszEcNOcBES4DwhhI4MyAfcKrWhTugPCoN5TOuaBZ5xQ1uFfBpYY=; Proxy: null), S3 Extended Request ID: /8BZO1eA4zZkeRJeVFnDlljCszEcNOcBES4DwhhI4MyAfcKrWhTugPCoN5TOuaBZ5xQ1uFfBpYY= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DQSH5CZWAH2JR3; S3 Extended Request ID: /8BZO1eA4zZkeRJeVFnDlljCszEcNOcBES4DwhhI4MyAfcKrWhTugPCoN5TOuaBZ5xQ1uFfBpYY=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T04:07:00,659 ERROR [Nuxeo-Work-default-91:2188893273296613.294792706] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DKZTSG2N05B534; S3 Extended Request ID: uJvMXR6wKAn7MMTWhEUjFPW0IZBcW7Ir7RWZcf/3nz7ulXiNW/KiXcDfIT/7FOplSmO+5cJRvko=; Proxy: null), S3 Extended Request ID: uJvMXR6wKAn7MMTWhEUjFPW0IZBcW7Ir7RWZcf/3nz7ulXiNW/KiXcDfIT/7FOplSmO+5cJRvko= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DKZTSG2N05B534; S3 Extended Request ID: uJvMXR6wKAn7MMTWhEUjFPW0IZBcW7Ir7RWZcf/3nz7ulXiNW/KiXcDfIT/7FOplSmO+5cJRvko=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T04:07:00,762 ERROR [Nuxeo-Work-default-91:2188893273296613.294792706] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DNEY5DKJZ9QXYK; S3 Extended Request ID: xH9H0Adz2CaUHpiLRdNz1auaGa62chTmBXMMVLOyJoZTG3uLHLvQm9I+HbQx77i/rGBi8lpo2XE=; Proxy: null), S3 Extended Request ID: xH9H0Adz2CaUHpiLRdNz1auaGa62chTmBXMMVLOyJoZTG3uLHLvQm9I+HbQx77i/rGBi8lpo2XE= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: T5DNEY5DKJZ9QXYK; S3 Extended Request ID: xH9H0Adz2CaUHpiLRdNz1auaGa62chTmBXMMVLOyJoZTG3uLHLvQm9I+HbQx77i/rGBi8lpo2XE=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T04:15:00,020 ERROR [Nuxeo-Work-default-200:2189373278770968.987822187] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T04:15:00,029 ERROR [Nuxeo-Work-default-200] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-200, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2189373278770968.987822187 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T04:30:00,014 ERROR [Nuxeo-Work-default-205:2190273271393996.2021165738] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T04:30:00,016 ERROR [Nuxeo-Work-default-205] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-205, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2190273271393996.2021165738 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T04:45:00,018 ERROR [Nuxeo-Work-default-206:2191173274078218.1675845682] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T04:45:00,026 ERROR [Nuxeo-Work-default-206] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-206, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2191173274078218.1675845682 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T05:00:00,015 ERROR [Nuxeo-Work-default-207:2192073272980713.657586409] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T05:00:00,019 ERROR [Nuxeo-Work-default-207] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-207, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2192073272980713.657586409 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T05:07:00,452 ERROR [Nuxeo-Work-default-92:2192493272791380.1284164296] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCNK8WDQEA64VM8; S3 Extended Request ID: nrg+rU6DTQ1bK5UhJV/FeV43DI6amjtCYuX1Za2ZHF1VqBCfEQD5wtTlaEmzwma71sKmX7hza8c=; Proxy: null), S3 Extended Request ID: nrg+rU6DTQ1bK5UhJV/FeV43DI6amjtCYuX1Za2ZHF1VqBCfEQD5wtTlaEmzwma71sKmX7hza8c= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCNK8WDQEA64VM8; S3 Extended Request ID: nrg+rU6DTQ1bK5UhJV/FeV43DI6amjtCYuX1Za2ZHF1VqBCfEQD5wtTlaEmzwma71sKmX7hza8c=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T05:07:00,554 ERROR [Nuxeo-Work-default-92:2192493272791380.1284164296] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCWVTZAXQPJX8T6; S3 Extended Request ID: G1ZT0L97j2HGAJYfd9nS2jEPjWIA8aCfbNpQjUbbsqkJi85Q5wdkblnvuhwLmb4/fvIcVmRk2vs=; Proxy: null), S3 Extended Request ID: G1ZT0L97j2HGAJYfd9nS2jEPjWIA8aCfbNpQjUbbsqkJi85Q5wdkblnvuhwLmb4/fvIcVmRk2vs= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCWVTZAXQPJX8T6; S3 Extended Request ID: G1ZT0L97j2HGAJYfd9nS2jEPjWIA8aCfbNpQjUbbsqkJi85Q5wdkblnvuhwLmb4/fvIcVmRk2vs=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T05:07:00,657 ERROR [Nuxeo-Work-default-92:2192493272791380.1284164296] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCK875TK8FSAQJ3; S3 Extended Request ID: gsQjUIK3AvgvYv/P/8xZb6A8l7yZucClv25XYZRNCNhf7EK0GVFCz5Fb8rXIy0uKinypLpTPuIE=; Proxy: null), S3 Extended Request ID: gsQjUIK3AvgvYv/P/8xZb6A8l7yZucClv25XYZRNCNhf7EK0GVFCz5Fb8rXIy0uKinypLpTPuIE= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCK875TK8FSAQJ3; S3 Extended Request ID: gsQjUIK3AvgvYv/P/8xZb6A8l7yZucClv25XYZRNCNhf7EK0GVFCz5Fb8rXIy0uKinypLpTPuIE=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T05:07:00,759 ERROR [Nuxeo-Work-default-92:2192493272791380.1284164296] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCG9Z5XD0GW6KPR; S3 Extended Request ID: uiTWLpLbURhnM7lscWgL/sv/WwIaXB09W2sR+x71a9PTEiyNk97/adTYCS76tdxzEqW0IffrGac=; Proxy: null), S3 Extended Request ID: uiTWLpLbURhnM7lscWgL/sv/WwIaXB09W2sR+x71a9PTEiyNk97/adTYCS76tdxzEqW0IffrGac= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: SMCG9Z5XD0GW6KPR; S3 Extended Request ID: uiTWLpLbURhnM7lscWgL/sv/WwIaXB09W2sR+x71a9PTEiyNk97/adTYCS76tdxzEqW0IffrGac=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T05:15:00,016 ERROR [Nuxeo-Work-default-204:2192973274384450.472259776] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T05:15:00,026 ERROR [Nuxeo-Work-default-204] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-204, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2192973274384450.472259776 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T05:30:00,015 ERROR [Nuxeo-Work-default-209:2193873272814151.859333614] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T05:30:00,024 ERROR [Nuxeo-Work-default-209] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-209, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2193873272814151.859333614 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T05:45:00,018 ERROR [Nuxeo-Work-default-210:2194773275472460.1274046804] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T05:45:00,021 ERROR [Nuxeo-Work-default-210] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-210, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2194773275472460.1274046804 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T06:00:00,015 ERROR [Nuxeo-Work-default-211:2195673271731648.1039374977] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T06:00:00,024 ERROR [Nuxeo-Work-default-211] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-211, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2195673271731648.1039374977 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T06:07:00,455 ERROR [Nuxeo-Work-default-91:2196093276607554.1409974165] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7F8T56GX623P4N; S3 Extended Request ID: DiFuwO6vpfSK0CUdBNje+6FiwzJIb4jNWLVHKTzoVMTCsBDwtEglhqbVOtr1ypfizuXCu0/bpUE=; Proxy: null), S3 Extended Request ID: DiFuwO6vpfSK0CUdBNje+6FiwzJIb4jNWLVHKTzoVMTCsBDwtEglhqbVOtr1ypfizuXCu0/bpUE= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7F8T56GX623P4N; S3 Extended Request ID: DiFuwO6vpfSK0CUdBNje+6FiwzJIb4jNWLVHKTzoVMTCsBDwtEglhqbVOtr1ypfizuXCu0/bpUE=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T06:07:00,558 ERROR [Nuxeo-Work-default-91:2196093276607554.1409974165] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ71DZWGGZRGVTQ3; S3 Extended Request ID: RvqKWKuvMM3MI/AYDngaMGDtdoo0gcHSVdhMWx+eCsDCIr7Otu1jydxXpubNXR4IYjrV7EUwqUU=; Proxy: null), S3 Extended Request ID: RvqKWKuvMM3MI/AYDngaMGDtdoo0gcHSVdhMWx+eCsDCIr7Otu1jydxXpubNXR4IYjrV7EUwqUU= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ71DZWGGZRGVTQ3; S3 Extended Request ID: RvqKWKuvMM3MI/AYDngaMGDtdoo0gcHSVdhMWx+eCsDCIr7Otu1jydxXpubNXR4IYjrV7EUwqUU=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T06:07:00,660 ERROR [Nuxeo-Work-default-91:2196093276607554.1409974165] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7DXV45F3HAEXM6; S3 Extended Request ID: S/5fGr6mukrTyCm3yIlm317efAxUNUMssQLtxZEOPS7ktxCb0NbV4W1rn3OakTnMefj/JPZKj5E=; Proxy: null), S3 Extended Request ID: S/5fGr6mukrTyCm3yIlm317efAxUNUMssQLtxZEOPS7ktxCb0NbV4W1rn3OakTnMefj/JPZKj5E= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7DXV45F3HAEXM6; S3 Extended Request ID: S/5fGr6mukrTyCm3yIlm317efAxUNUMssQLtxZEOPS7ktxCb0NbV4W1rn3OakTnMefj/JPZKj5E=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T06:07:00,762 ERROR [Nuxeo-Work-default-91:2196093276607554.1409974165] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7DFX2AG75AJJG1; S3 Extended Request ID: Z35oiESf7DZYbZvPTEW3CIbLfH5H+gmd7p4OlN3bKzhPRaEEcg/+b//E/7cqWUnO185QN6oQhLw=; Proxy: null), S3 Extended Request ID: Z35oiESf7DZYbZvPTEW3CIbLfH5H+gmd7p4OlN3bKzhPRaEEcg/+b//E/7cqWUnO185QN6oQhLw= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: BZ7DFX2AG75AJJG1; S3 Extended Request ID: Z35oiESf7DZYbZvPTEW3CIbLfH5H+gmd7p4OlN3bKzhPRaEEcg/+b//E/7cqWUnO185QN6oQhLw=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T06:15:00,016 ERROR [Nuxeo-Work-default-208:2196573274085371.2020247799] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T06:15:00,018 ERROR [Nuxeo-Work-default-208] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-208, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2196573274085371.2020247799 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T06:30:00,032 ERROR [Nuxeo-Work-default-213:2197473289779961.318158680] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T06:30:00,040 ERROR [Nuxeo-Work-default-213] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-213, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2197473289779961.318158680 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T06:45:00,018 ERROR [Nuxeo-Work-default-214:2198373275446426.773289165] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T06:45:00,020 ERROR [Nuxeo-Work-default-214] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-214, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2198373275446426.773289165 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T07:00:00,017 ERROR [Nuxeo-Work-default-215:2199273274572958.1245438356] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T07:00:00,026 ERROR [Nuxeo-Work-default-215] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-215, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2199273274572958.1245438356 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T07:07:00,470 ERROR [Nuxeo-Work-default-92:2199693274306702.823739984] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b0fe8c07-f4e7-47e8-ab2c-fdee6cd37a35, path=null, title=sample1) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHNF6MZ4XKDZEGG; S3 Extended Request ID: nBAqm6cV8gr1PUp/LVMCbOYRAbzPaf8jQ5kQP8QmF0ijAJ9iinTuC9Ik7t5ZqEVhTxGRVOEVe6g=; Proxy: null), S3 Extended Request ID: nBAqm6cV8gr1PUp/LVMCbOYRAbzPaf8jQ5kQP8QmF0ijAJ9iinTuC9Ik7t5ZqEVhTxGRVOEVe6g= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHNF6MZ4XKDZEGG; S3 Extended Request ID: nBAqm6cV8gr1PUp/LVMCbOYRAbzPaf8jQ5kQP8QmF0ijAJ9iinTuC9Ik7t5ZqEVhTxGRVOEVe6g=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T07:07:00,572 ERROR [Nuxeo-Work-default-92:2199693274306702.823739984] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(b479106c-9094-4b4d-b65c-b16ba3aec5ec, path=null, title=sample2) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHS8TK93K32HBTK; S3 Extended Request ID: 1EyBmmyAxB9j6hieWs+CNboBL7gReTaTU130i4U5qb6UKQuGkdf0dcUg2Rp2H2lt+i703AxFEQ8=; Proxy: null), S3 Extended Request ID: 1EyBmmyAxB9j6hieWs+CNboBL7gReTaTU130i4U5qb6UKQuGkdf0dcUg2Rp2H2lt+i703AxFEQ8= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHS8TK93K32HBTK; S3 Extended Request ID: 1EyBmmyAxB9j6hieWs+CNboBL7gReTaTU130i4U5qb6UKQuGkdf0dcUg2Rp2H2lt+i703AxFEQ8=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T07:07:00,675 ERROR [Nuxeo-Work-default-92:2199693274306702.823739984] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(e9ac68bd-aa1a-4ad3-996d-778046b5de90, path=null, title=sample3) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHHXZRNFV8ENJNZ; S3 Extended Request ID: NEN08Ol6E0XjFLbJRHTBLIs+hO3GWEgz/2IhOj2Ix/Pn1wVMgjtVGawdOo1R1HMtXMUNhtW0YZ0=; Proxy: null), S3 Extended Request ID: NEN08Ol6E0XjFLbJRHTBLIs+hO3GWEgz/2IhOj2Ix/Pn1wVMgjtVGawdOo1R1HMtXMUNhtW0YZ0= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHHXZRNFV8ENJNZ; S3 Extended Request ID: NEN08Ol6E0XjFLbJRHTBLIs+hO3GWEgz/2IhOj2Ix/Pn1wVMgjtVGawdOo1R1HMtXMUNhtW0YZ0=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T07:07:00,777 ERROR [Nuxeo-Work-default-92:2199693274306702.823739984] [org.nuxeo.coldstorage.service.ColdStorageServiceImpl] Unable to get the cold storage blob status for document: DocumentModelImpl(6e34ce93-785a-4cc1-ad8f-1d66078b4f92, path=/devops/workspaces/aba/testGlacier, title=testGlacier) java.io.IOException: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHQCJH740QMY4YS; S3 Extended Request ID: 9EykTIAbdWu/qcUcoUu+wAymSd8p0hqqSThvN4bdvO7GAVAo2znHYBm9DLch9SIyAiGn80hgXOU=; Proxy: null), S3 Extended Request ID: 9EykTIAbdWu/qcUcoUu+wAymSd8p0hqqSThvN4bdvO7GAVAo2znHYBm9DLch9SIyAiGn80hgXOU= at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:232) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: 9XHQCJH740QMY4YS; S3 Extended Request ID: 9EykTIAbdWu/qcUcoUu+wAymSd8p0hqqSThvN4bdvO7GAVAo2znHYBm9DLch9SIyAiGn80hgXOU=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) ~[nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] ... 10 more 2021-09-27T07:15:00,016 ERROR [Nuxeo-Work-default-212:2200173274914114.984618414] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T07:15:00,023 ERROR [Nuxeo-Work-default-212] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-212, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2200173274914114.984618414 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T07:30:00,016 ERROR [Nuxeo-Work-default-217:2201073273140741.1232921524] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T07:30:00,019 ERROR [Nuxeo-Work-default-217] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-217, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2201073273140741.1232921524 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T07:45:00,015 ERROR [Nuxeo-Work-default-218:2201973273177279.319170236] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: TransiantStorageGCWork(, Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T07:45:00,025 ERROR [Nuxeo-Work-default-218] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-218, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 0 retries, class=class org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork id=2201973273177279.319170236 category=TransiantStorageGCWork title=Transient Store GC at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.NuxeoException: Already started at org.nuxeo.ecm.core.blob.AbstractBlobGarbageCollector.start(AbstractBlobGarbageCollector.java:52) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.blob.CachingBlobStore$CachingBinaryGarbageCollector.start(CachingBlobStore.java:469) ~[nuxeo-core-api-10.10-HF52.jar:?] at org.nuxeo.ecm.core.transientstore.keyvalueblob.KeyValueBlobTransientStore.doGC(KeyValueBlobTransientStore.java:280) ~[nuxeo-core-cache-10.10-HF48.jar:?] at java.util.HashMap$Values.forEach(HashMap.java:981) ~[?:1.8.0_302] at org.nuxeo.ecm.core.transientstore.TransientStorageComponent.doGC(TransientStorageComponent.java:98) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.transientstore.TransiantStorageGCWork.work(TransiantStorageGCWork.java:41) ~[nuxeo-core-cache-10.10-HF48.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more 2021-09-27T07:55:51,363 INFO [main] [org.nuxeo.osgi.application.loader.FrameworkLoader] ====================================================================== = Starting Nuxeo Framework ====================================================================== * Distribution Version = 10.10 * Distribution Date = 201901211253 * Server home = /opt/nuxeo/server * Runtime home = /opt/nuxeo/server/nxserver * Data Directory = /var/lib/nuxeo/data * Log Directory = /var/log/nuxeo * Configuration Directory = /opt/nuxeo/server/nxserver/config * Temp Directory = /opt/nuxeo/server/tmp ====================================================================== 2021-09-27T07:56:07,775 WARN [main] [org.nuxeo.ecm.platform.commandline.executor.service.CommandLineExecutorComponent] Command not available: videoClosedCaptionsExtractor (command ccextractor not found in system path (descriptor CommandLineDescriptor[name=videoClosedCaptionsExtractor,enabled=true,available=false,command=ccextractor,parameterString=#{sourceFilePath} -out=#{outFormat} -trim -o #{outFilePath},testParameterString=,winParameterString=,winTestParameterString=,winCommand=,tester=,readOutput=true,installationDirective=You need to install ccextractor.,installErrorMessage=,timeout=]). You need to install ccextractor.) 2021-09-27T07:56:07,779 WARN [main] [org.nuxeo.ecm.platform.commandline.executor.service.CommandLineExecutorComponent] Command not available: videoPartClosedCaptionsExtractor (command ccextractor not found in system path (descriptor CommandLineDescriptor[name=videoPartClosedCaptionsExtractor,enabled=true,available=false,command=ccextractor,parameterString=#{sourceFilePath} -out=#{outFormat} -startat #{startAt} -endat #{endAt} -trim -o #{outFilePath},testParameterString=,winParameterString=,winTestParameterString=,winCommand=,tester=,readOutput=true,installationDirective=You need to install ccextractor.,installErrorMessage=,timeout=]). You need to install ccextractor.) 2021-09-27T07:56:10,872 WARN [main] [org.nuxeo.ecm.directory.DirectoryRegistry] Directory template-directory redefined with different factory 2021-09-27T07:56:19,073 WARN [main] [org.nuxeo.elasticsearch.core.ElasticSearchEmbeddedNode] Elasticsearch embedded configuration is ONLY for testing purpose. You need to create a dedicated Elasticsearch cluster for production. 2021-09-27T07:56:28,163 ERROR [Quartz_Scheduler] [org.quartz.core.QuartzSchedulerThread] quartzSchedulerThreadLoop: RuntimeException null java.lang.NullPointerException: null at com.novemberain.quartz.mongodb.dao.JobDao.requestsRecovery(JobDao.java:104) ~[quartz-mongodb-2.0.0-NX3.jar:?] at com.novemberain.quartz.mongodb.cluster.TriggerRecoverer.doRecovery(TriggerRecoverer.java:65) ~[quartz-mongodb-2.0.0-NX3.jar:?] at com.novemberain.quartz.mongodb.TriggerRunner.acquireNextTriggers(TriggerRunner.java:127) ~[quartz-mongodb-2.0.0-NX3.jar:?] at com.novemberain.quartz.mongodb.TriggerRunner.acquireNext(TriggerRunner.java:66) ~[quartz-mongodb-2.0.0-NX3.jar:?] at com.novemberain.quartz.mongodb.MongoDBJobStore.acquireNextTriggers(MongoDBJobStore.java:398) ~[quartz-mongodb-2.0.0-NX3.jar:?] at org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:287) [quartz-2.3.0.jar:?] 2021-09-27T07:56:30,944 INFO [main] [org.nuxeo.runtime.osgi.OSGiRuntimeService] Nuxeo Platform Started ====================================================================== = Component Loading Status: Pending: 0 / Missing: 0 / Unstarted: 0 / Total: 801 ====================================================================== 2021-09-27T07:56:39,684 INFO [main] [javax.enterprise.resource.webcontainer.jsf.application] jsf.core.injection.provider_generic_web_configured 2021-09-27T08:07:00,510 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.blob.s3.S3BlobProvider] Failed to get information on blob: default:e2a1ff6f208b696a3c0714d1e2582d58 com.amazonaws.services.s3.model.AmazonS3Exception: Not Found (Service: Amazon S3; Status Code: 404; Error Code: 404 Not Found; Request ID: PTWFRFEZ4EQB8PJ8; S3 Extended Request ID: 6IZqG4td+CKstU3HXec5gzxpRkPUPQt1vC5erEeTUuXRV5BYPtVK9Jy9gu+eFxCG2FmO+4cS9GE=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) [nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,521 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: ListenerWork(Listener checkColdStorageClassUpdated [checkColdStorageClassUpdated], Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,521 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.event.impl.AsyncEventExecutor] Failed to execute async event null on listener checkColdStorageClassUpdated org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,646 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.blob.s3.S3BlobProvider] Failed to get information on blob: default:e2a1ff6f208b696a3c0714d1e2582d58 com.amazonaws.services.s3.model.AmazonS3Exception: Not Found (Service: Amazon S3; Status Code: 404; Error Code: 404 Not Found; Request ID: PTW0AVJGZ080PACH; S3 Extended Request ID: VJ9wB4kN7fGNl06zcms7cLOuZLuwGNM/O/Ti42ISmLsBO7zxpuCVNWnP7i/W/q+zB8+qnEFfY3g=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) [nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,652 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: ListenerWork(Listener checkColdStorageClassUpdated [checkColdStorageClassUpdated], Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,653 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.event.impl.AsyncEventExecutor] Failed to execute async event null on listener checkColdStorageClassUpdated org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,771 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.blob.s3.S3BlobProvider] Failed to get information on blob: default:e2a1ff6f208b696a3c0714d1e2582d58 com.amazonaws.services.s3.model.AmazonS3Exception: Not Found (Service: Amazon S3; Status Code: 404; Error Code: 404 Not Found; Request ID: PTW1RF8C7R6TZ1C9; S3 Extended Request ID: hKUHsGWEZ1zoqSteSnYvnFIV2CyZGzfOeDGrLKtnsq2wkMiKohg6m+L+SoV3jOCxEroVo/b613s=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1828) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1412) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1374) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1145) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:802) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:704) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:550) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:530) ~[aws-java-sdk-core-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5219) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5165) ~[aws-java-sdk-s3-1.11.878.jar:?] at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1350) ~[aws-java-sdk-s3-1.11.878.jar:?] at org.nuxeo.ecm.blob.s3.S3BlobProvider.getStatus(S3BlobProvider.java:225) [nuxeo-core-binarymanager-s3-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.getBlobStatus(ColdStorageServiceImpl.java:130) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:460) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) [nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,777 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.work.AbstractWork] Exception during work: ListenerWork(Listener checkColdStorageClassUpdated [checkColdStorageClassUpdated], Progress(?%, ?/0), null) org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,778 ERROR [Nuxeo-Work-default-1:2203293279474625.1888476713] [org.nuxeo.ecm.core.event.impl.AsyncEventExecutor] Failed to execute async event null on listener checkColdStorageClassUpdated org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) [nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) [nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] 2021-09-27T08:07:00,808 ERROR [Nuxeo-Work-default-1] [org.nuxeo.ecm.core.work.WorkManagerImpl] Uncaught error on thread: Nuxeo-Work-default-1, current work might be lost, WorkManager metrics might be corrupted. org.nuxeo.ecm.core.api.NuxeoException: Work failed after 2 retries, class=class org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork id=2203293279474625.1888476713 category=checkColdStorageClassUpdated title=Listener checkColdStorageClassUpdated [checkColdStorageClassUpdated] at org.nuxeo.ecm.core.work.AbstractWork.workFailed(AbstractWork.java:439) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:395) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Caused by: org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] ... 4 more Suppressed: org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302] Suppressed: org.nuxeo.ecm.core.api.PropertyException: Cannot set property on a version: coldstorage:classStorageToUpdate at org.nuxeo.ecm.core.storage.BaseDocument.checkReadOnlyIgnoredWrite(BaseDocument.java:303) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeComplexProperty(BaseDocument.java:881) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.BaseDocument.writeDocumentPart(BaseDocument.java:848) ~[nuxeo-core-storage-10.10-HF49.jar:?] at org.nuxeo.ecm.core.storage.dbs.DBSDocument.writeDocumentPart(DBSDocument.java:1089) ~[nuxeo-core-storage-dbs-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.DocumentModelFactory.writeDocumentModel(DocumentModelFactory.java:257) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:374) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.writeModel(AbstractSession.java:370) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.ecm.core.api.AbstractSession.saveDocument(AbstractSession.java:1611) ~[nuxeo-core-10.10-HF52.jar:?] at org.nuxeo.coldstorage.service.ColdStorageServiceImpl.checkColdStorageClass(ColdStorageServiceImpl.java:468) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.coldstorage.events.CheckColdStorageClassUpdatedListener.handleEvent(CheckColdStorageClassUpdatedListener.java:51) ~[nuxeo-coldstorage-10.0.0-rc.28.jar:?] at org.nuxeo.ecm.core.event.impl.AsyncEventExecutor$ListenerWork.work(AsyncEventExecutor.java:221) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.runWorkWithTransaction(AbstractWork.java:493) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.AbstractWork.run(AbstractWork.java:383) ~[nuxeo-core-event-10.10-HF44.jar:?] at org.nuxeo.ecm.core.work.WorkHolder.run(WorkHolder.java:57) ~[nuxeo-core-event-10.10-HF44.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_302] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_302] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_302]