[2020-05-16T02:22:38,049][INFO ][o.e.c.r.a.DiskThresholdMonitor] [DYUbwoP] low disk watermark [85%] exceeded on [DYUbwoPHSAWB08omk4MmEA][DYUbwoP][/opt/elasticsearch-6.8.0/data/nodes/0] free: 929.3mb[14.6%], replicas will not be assigned to this node
当磁盘空间占用超过90%时，Elasticsearch输出如下日志：
1 2
[2020-05-16T02:26:38,124][WARN ][o.e.c.r.a.DiskThresholdMonitor] [DYUbwoP] high disk watermark [90%] exceeded on [DYUbwoPHSAWB08omk4MmEA][DYUbwoP][/opt/elasticsearch-6.8.0/data/nodes/0] free: 599.9mb[9.4%], shards will be relocated away from this node
[2020-05-16T02:27:38,155][WARN ][o.e.c.r.a.DiskThresholdMonitor] [DYUbwoP] flood stage disk watermark [95%] exceeded on [DYUbwoPHSAWB08omk4MmEA][DYUbwoP][/opt/elasticsearch-6.8.0/data/nodes/0] free: 316.1mb[4.9%], all indices on this node will be marked read-only [2020-05-16T02:29:11,336][WARN ][o.e.x.m.e.l.LocalExporter] [DYUbwoP] unexpected error while indexing monitoring document org.elasticsearch.xpack.monitoring.exporter.ExportException: ClusterBlockException[blocked by: [FORBIDDEN/12/index read-only / allow delete (api)];] at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$throwExportException$2(LocalBulk.java:125) ~[?:?] at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_161] at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_161] at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_161] at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_161] at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_161] at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) ~[?:1.8.0_161] at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) ~[?:1.8.0_161] at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_161] at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_161] at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:126) ~[?:?] at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:108) ~[?:?] at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:61) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:665) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:61) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:470) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:465) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:945) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:785) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:171) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:99) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:167) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:121) ~[?:?] at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:165) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:139) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:81) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation.doRun(TransportBulkAction.java:440) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction.executeBulk(TransportBulkAction.java:553) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:256) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.bulk.TransportBulkAction.lambda$processBulkIndexIngestRequest$4(TransportBulkAction.java:607) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.ingest.IngestService$4.doRun(IngestService.java:411) [elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.8.0.jar:6.8.0] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161] Caused by: org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [FORBIDDEN/12/index read-only / allow delete (api)]; at org.elasticsearch.cluster.block.ClusterBlocks.indexBlockedException(ClusterBlocks.java:208) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction.blockExceptions(TransportReplicationAction.java:254) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:99) ~[elasticsearch-6.8.0.jar:6.8.0] at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:779) ~[elasticsearch-6.8.0.jar:6.8.0] ... 19 more
解决
紧急给Elasticsearch的硬盘扩容，扩容完毕后执行以下语句关闭索引的只读状态：
1 2 3 4
PUT _all/_settings { "index.blocks.read_only_allow_delete": null }
