From c001dcda60d00897ad9de3a3cad232c3e01fd408 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Wed, 20 Sep 2023 21:23:56 +0800 Subject: [PATCH 1/7] MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. --- .../app/job/impl/TestMapReduceChildJVM.java | 58 +++++++++---------- .../org/apache/hadoop/mapred/TaskLog.java | 3 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +- .../src/main/resources/mapred-default.xml | 7 +-- .../launcher/TestContainerLaunch.java | 2 +- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index e22bd44ee905b..5216a2f523486 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -58,20 +58,20 @@ public void testCommandLine() throws Exception { app.verifyCompleted(); Assert.assertEquals( - "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + - " -Djava.net.preferIPv4Stack=true" + - " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + - " -Dlog4j.configuration=container-log4j.properties" + - " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=0" + - " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + - " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + - " 54321" + - " attempt_0_0000_m_000000_0" + - " 0" + - " 1>/stdout" + - " 2>/stderr ]", app.launchCmdList.get(0)); + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN " + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=container-log4j.properties" + + " -Dyarn.app.container.log.dir=" + + " -Dyarn.app.container.log.filesize=10485760" + + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1>/stdout" + + " 2>/stderr ]", app.launchCmdList.get(0)); Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); @@ -127,7 +127,7 @@ private void testReduceCommandLine(Configuration conf) " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=0" + + " -Dyarn.app.container.log.filesize=10485760" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + @@ -163,20 +163,20 @@ public void testCommandLineWithLog4JConifg() throws Exception { app.verifyCompleted(); Assert.assertEquals( - "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + - " -Djava.net.preferIPv4Stack=true" + - " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + - " -Dlog4j.configuration=" + testLogPropertieFile + - " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=0" + - " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + - " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + - " 54321" + - " attempt_0_0000_m_000000_0" + - " 0" + - " 1>/stdout" + - " 2>/stderr ]", app.launchCmdList.get(0)); + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN " + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=" + testLogPropertieFile + + " -Dyarn.app.container.log.dir=" + + " -Dyarn.app.container.log.filesize=10485760" + + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1>/stdout" + + " 2>/stderr ]", app.launchCmdList.get(0)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index a0223dedd64ce..fdfa3e7710f47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -466,7 +466,8 @@ public static long getTaskLogLength(JobConf conf) { } public static long getTaskLogLimitBytes(Configuration conf) { - return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; + return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) * + 1024; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 8ec984e777bb6..24689e09daf13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -410,6 +410,8 @@ public interface MRJobConfig { public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; + public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240; + public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; @@ -758,11 +760,11 @@ public interface MRJobConfig { public static final String MR_AM_LOG_KB = MR_AM_PREFIX + "container.log.limit.kb"; - public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll + public static final int DEFAULT_MR_AM_LOG_KB = 10240; public static final String MR_AM_LOG_BACKUPS = MR_AM_PREFIX + "container.log.backups"; - public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; + public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a6d68acda34ab..493e4c3a37e94 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -823,16 +823,15 @@ mapreduce.task.userlog.limit.kb - 0 - The maximum size of user-logs of each task in KB. 0 disables the cap. + 10240 + The maximum size of user-logs of each task in KB. yarn.app.mapreduce.am.container.log.limit.kb - 0 + 10240 The maximum size of the MRAppMaster attempt container logs in KB. - 0 disables the cap. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index bd135ff519382..b45b785f5a53b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -945,7 +945,7 @@ private void verifyTailErrorLogOnContainerExit(Configuration conf, invalidCommand.add("-Djava.io.tmpdir=$PWD/tmp"); invalidCommand.add("-Dlog4j.configuration=container-log4j.properties"); invalidCommand.add("-Dyarn.app.container.log.dir=" + containerLogDir); - invalidCommand.add("-Dyarn.app.container.log.filesize=0"); + invalidCommand.add("-Dyarn.app.container.log.filesize=10485760"); invalidCommand.add("-Dhadoop.root.logger=INFO,CLA"); invalidCommand.add("-Dhadoop.root.logfile=syslog"); invalidCommand.add("-Xmx1024m"); From ceba05a0aaeba5ec8efc55bf9cf670347c19443d Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 30 Sep 2023 22:20:51 +0800 Subject: [PATCH 2/7] Revert "MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0." This reverts commit c001dcda60d00897ad9de3a3cad232c3e01fd408. --- .../app/job/impl/TestMapReduceChildJVM.java | 58 +++++++++---------- .../org/apache/hadoop/mapred/TaskLog.java | 3 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +- .../src/main/resources/mapred-default.xml | 7 ++- .../launcher/TestContainerLaunch.java | 2 +- 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index 5216a2f523486..e22bd44ee905b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -58,20 +58,20 @@ public void testCommandLine() throws Exception { app.verifyCompleted(); Assert.assertEquals( - "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + - " -Djava.net.preferIPv4Stack=true" + - " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + - " -Dlog4j.configuration=container-log4j.properties" + - " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=10485760" + - " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + - " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + - " 54321" + - " attempt_0_0000_m_000000_0" + - " 0" + - " 1>/stdout" + - " 2>/stderr ]", app.launchCmdList.get(0)); + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN " + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=container-log4j.properties" + + " -Dyarn.app.container.log.dir=" + + " -Dyarn.app.container.log.filesize=0" + + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1>/stdout" + + " 2>/stderr ]", app.launchCmdList.get(0)); Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); @@ -127,7 +127,7 @@ private void testReduceCommandLine(Configuration conf) " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=10485760" + + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + @@ -163,20 +163,20 @@ public void testCommandLineWithLog4JConifg() throws Exception { app.verifyCompleted(); Assert.assertEquals( - "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + - " -Djava.net.preferIPv4Stack=true" + - " -Dhadoop.metrics.log.level=WARN " + - " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + - " -Dlog4j.configuration=" + testLogPropertieFile + - " -Dyarn.app.container.log.dir=" + - " -Dyarn.app.container.log.filesize=10485760" + - " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + - " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + - " 54321" + - " attempt_0_0000_m_000000_0" + - " 0" + - " 1>/stdout" + - " 2>/stderr ]", app.launchCmdList.get(0)); + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN " + + " -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=" + testLogPropertieFile + + " -Dyarn.app.container.log.dir=" + + " -Dyarn.app.container.log.filesize=0" + + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1>/stdout" + + " 2>/stderr ]", app.launchCmdList.get(0)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index fdfa3e7710f47..a0223dedd64ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -466,8 +466,7 @@ public static long getTaskLogLength(JobConf conf) { } public static long getTaskLogLimitBytes(Configuration conf) { - return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) * - 1024; + return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 24689e09daf13..8ec984e777bb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -410,8 +410,6 @@ public interface MRJobConfig { public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; - public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240; - public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; @@ -760,11 +758,11 @@ public interface MRJobConfig { public static final String MR_AM_LOG_KB = MR_AM_PREFIX + "container.log.limit.kb"; - public static final int DEFAULT_MR_AM_LOG_KB = 10240; + public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll public static final String MR_AM_LOG_BACKUPS = MR_AM_PREFIX + "container.log.backups"; - public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll + public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; /**The number of splits when reporting progress in MR*/ public static final String MR_AM_NUM_PROGRESS_SPLITS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 493e4c3a37e94..a6d68acda34ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -823,15 +823,16 @@ mapreduce.task.userlog.limit.kb - 10240 - The maximum size of user-logs of each task in KB. + 0 + The maximum size of user-logs of each task in KB. 0 disables the cap. yarn.app.mapreduce.am.container.log.limit.kb - 10240 + 0 The maximum size of the MRAppMaster attempt container logs in KB. + 0 disables the cap. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index b45b785f5a53b..bd135ff519382 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -945,7 +945,7 @@ private void verifyTailErrorLogOnContainerExit(Configuration conf, invalidCommand.add("-Djava.io.tmpdir=$PWD/tmp"); invalidCommand.add("-Dlog4j.configuration=container-log4j.properties"); invalidCommand.add("-Dyarn.app.container.log.dir=" + containerLogDir); - invalidCommand.add("-Dyarn.app.container.log.filesize=10485760"); + invalidCommand.add("-Dyarn.app.container.log.filesize=0"); invalidCommand.add("-Dhadoop.root.logger=INFO,CLA"); invalidCommand.add("-Dhadoop.root.logfile=syslog"); invalidCommand.add("-Xmx1024m"); From 22141d5133a61cf5cf6ee288ba7e72b289904b74 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 30 Sep 2023 22:21:36 +0800 Subject: [PATCH 3/7] MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0. --- .../src/main/resources/container-log4j.properties | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index c5371c6d9ef4f..095409697c987 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -30,10 +30,9 @@ yarn.app.container.log.filesize=100MB yarn.app.container.log.backups=1 yarn.app.mapreduce.shuffle.log.backups=1 -log4j.appender.CLA=org.apache.log4j.RollingFileAppender +log4j.appender.CLA=org.apache.log4j.FileAppender log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize} -log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CLA.Append=true log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n From df40fa27251dbc8fe822c8f9d78b8444eb37bbe2 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 30 Sep 2023 23:59:57 +0800 Subject: [PATCH 4/7] Revert "MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0." This reverts commit 22141d5133a61cf5cf6ee288ba7e72b289904b74. --- .../src/main/resources/container-log4j.properties | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index 095409697c987..c5371c6d9ef4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -30,9 +30,10 @@ yarn.app.container.log.filesize=100MB yarn.app.container.log.backups=1 yarn.app.mapreduce.shuffle.log.backups=1 -log4j.appender.CLA=org.apache.log4j.FileAppender +log4j.appender.CLA=org.apache.log4j.RollingFileAppender log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CLA.Append=true +log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize} +log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups} log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n From 9e56bf35104727c5b3a1adb344b678dd62e0fa00 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sun, 1 Oct 2023 00:00:30 +0800 Subject: [PATCH 5/7] Revert "HADOOP-18649. CLA and CRLA appenders to be replaced with RFA (#5448)" This reverts commit cf4a678ce946cecd035677bce756794ef286d795. --- .../hadoop/mapreduce/v2/util/MRApps.java | 5 +- .../src/main/resources/mapred-default.xml | 15 +- .../hadoop/yarn/ContainerLogAppender.java | 129 ++++++++++++++++++ .../yarn/ContainerRollingLogAppender.java | 76 +++++++++++ .../hadoop/yarn/TestContainerLogAppender.java | 48 +++++++ .../main/resources/container-log4j.properties | 39 +++--- 6 files changed, 288 insertions(+), 24 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 72dd48b09c2cb..a3ccfd72d8ce8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -60,6 +60,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.ApplicationClassLoader; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.ContainerLogAppender; +import org.apache.hadoop.yarn.ContainerRollingLogAppender; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -586,7 +588,8 @@ public static String getChildLogLevel(Configuration conf, boolean isMap) { /** * Add the JVM system properties necessary to configure - * {@link org.apache.log4j.RollingFileAppender}. + * {@link ContainerLogAppender} or + * {@link ContainerRollingLogAppender}. * * @param task for map/reduce, or null for app master * @param vargs the argument list to append to diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a6d68acda34ab..9b0d8b563d7bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -840,8 +840,11 @@ yarn.app.mapreduce.task.container.log.backups 0 Number of backup files for task logs when using - RollingFileAppender (RFA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for tasks when both mapreduce.task.userlog.limit.kb and + yarn.app.mapreduce.task.container.log.backups are greater than zero. @@ -849,8 +852,12 @@ yarn.app.mapreduce.am.container.log.backups 0 Number of backup files for the ApplicationMaster logs when using - RollingFileAppender (RFA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for the ApplicationMaster when both + yarn.app.mapreduce.am.container.log.limit.kb and + yarn.app.mapreduce.am.container.log.backups are greater than zero. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java new file mode 100644 index 0000000000000..09efe41e0c22f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import java.io.File; +import java.io.Flushable; +import java.util.ArrayDeque; +import java.util.Deque; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.log4j.FileAppender; +import org.apache.log4j.spi.LoggingEvent; + +/** + * A simple log4j-appender for container's logs. + */ +@Public +@Unstable +public class ContainerLogAppender extends FileAppender + implements Flushable { + + private String containerLogDir; + private String containerLogFile; + private int maxEvents; + private Deque eventBuffer; + private boolean closed = false; + + @Override + public synchronized void activateOptions() { + if (maxEvents > 0) { + this.eventBuffer = new ArrayDeque<>(); + } + setFile(new File(this.containerLogDir, containerLogFile).toString()); + setAppend(true); + super.activateOptions(); + } + + @Override + public synchronized void append(LoggingEvent event) { + if (closed) { + return; + } + if (eventBuffer != null) { + if (eventBuffer.size() == maxEvents) { + eventBuffer.removeFirst(); + } + eventBuffer.addLast(event); + } else { + super.append(event); + } + } + + @Override + public void flush() { + if (qw != null) { + qw.flush(); + } + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + if (eventBuffer != null) { + for (LoggingEvent event : eventBuffer) { + super.append(event); + } + // let garbage collection do its work + eventBuffer = null; + } + super.close(); + } + } + + /** + * Getter/Setter methods for log4j. + * + * @return containerLogDir. + */ + + public String getContainerLogDir() { + return this.containerLogDir; + } + + public void setContainerLogDir(String containerLogDir) { + this.containerLogDir = containerLogDir; + } + + public String getContainerLogFile() { + return containerLogFile; + } + + public void setContainerLogFile(String containerLogFile) { + this.containerLogFile = containerLogFile; + } + + private static final long EVENT_SIZE = 100; + + public long getTotalLogFileSize() { + return maxEvents * EVENT_SIZE; + } + + /** + * Setter so that log4j can configure it from the + * configuration(log4j.properties). + * + * @param logSize log size. + */ + public void setTotalLogFileSize(long logSize) { + maxEvents = (int)(logSize / EVENT_SIZE); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java new file mode 100644 index 0000000000000..f0e00fc194064 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.log4j.RollingFileAppender; + +import java.io.File; +import java.io.Flushable; + +/** + * A simple log4j-appender for container's logs. + * + */ +@Public +@Unstable +public class ContainerRollingLogAppender extends RollingFileAppender + implements Flushable { + private String containerLogDir; + private String containerLogFile; + + @Override + public void activateOptions() { + synchronized (this) { + setFile(new File(this.containerLogDir, containerLogFile).toString()); + setAppend(true); + super.activateOptions(); + } + } + + @Override + public void flush() { + if (qw != null) { + qw.flush(); + } + } + + /** + * Getter/Setter methods for log4j. + * + * @return containerLogDir. + */ + + public String getContainerLogDir() { + return this.containerLogDir; + } + + public void setContainerLogDir(String containerLogDir) { + this.containerLogDir = containerLogDir; + } + + public String getContainerLogFile() { + return containerLogFile; + } + + public void setContainerLogFile(String containerLogFile) { + this.containerLogFile = containerLogFile; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java new file mode 100644 index 0000000000000..26acfd7bad87b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn; + +import org.junit.jupiter.api.Test; + +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; + +public class TestContainerLogAppender { + + @Test + void testAppendInClose() throws Exception { + final ContainerLogAppender claAppender = new ContainerLogAppender(); + claAppender.setName("testCLA"); + claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n")); + claAppender.setContainerLogDir("target/testAppendInClose/logDir"); + claAppender.setContainerLogFile("syslog"); + claAppender.setTotalLogFileSize(1000); + claAppender.activateOptions(); + final Logger claLog = Logger.getLogger("testAppendInClose-catergory"); + claLog.setAdditivity(false); + claLog.addAppender(claAppender); + claLog.info(new Object() { + public String toString() { + claLog.info("message1"); + return "return message1"; + } + }); + claAppender.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index c5371c6d9ef4f..678e3a74c897a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -26,35 +26,36 @@ log4j.threshold=ALL # #Default values -yarn.app.container.log.filesize=100MB -yarn.app.container.log.backups=1 -yarn.app.mapreduce.shuffle.log.backups=1 +yarn.app.container.log.dir=null +yarn.app.container.log.filesize=100 -log4j.appender.CLA=org.apache.log4j.RollingFileAppender -log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize} -log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender +log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.CLA.containerLogFile=${hadoop.root.logfile} +log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize} log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.CRLA=org.apache.log4j.RollingFileAppender -log4j.appender.CRLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} -log4j.appender.CRLA.MaxFileSize=${yarn.app.container.log.filesize} -log4j.appender.CRLA.MaxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender +log4j.appender.CRLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.CRLA.containerLogFile=${hadoop.root.logfile} +log4j.appender.CRLA.maximumFileSize=${yarn.app.container.log.filesize} +log4j.appender.CRLA.maxBackupIndex=${yarn.app.container.log.backups} log4j.appender.CRLA.layout=org.apache.log4j.PatternLayout log4j.appender.CRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCLA=org.apache.log4j.RollingFileAppender -log4j.appender.shuffleCLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} -log4j.appender.shuffleCLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} +log4j.appender.shuffleCLA=org.apache.hadoop.yarn.ContainerLogAppender +log4j.appender.shuffleCLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.shuffleCLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCLA.totalLogFileSize=${yarn.app.mapreduce.shuffle.log.filesize} log4j.appender.shuffleCLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCRLA=org.apache.log4j.RollingFileAppender -log4j.appender.shuffleCRLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCRLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} -log4j.appender.shuffleCRLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} +log4j.appender.shuffleCRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender +log4j.appender.shuffleCRLA.containerLogDir=${yarn.app.container.log.dir} +log4j.appender.shuffleCRLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCRLA.maximumFileSize=${yarn.app.mapreduce.shuffle.log.filesize} +log4j.appender.shuffleCRLA.maxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} log4j.appender.shuffleCRLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n From 63c519494bf53399759a51bf01083d2d22772c12 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sun, 1 Oct 2023 07:43:22 +0800 Subject: [PATCH 6/7] remove blank --- .../main/java/org/apache/hadoop/yarn/ContainerLogAppender.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java index 09efe41e0c22f..03a0078167f80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java @@ -94,7 +94,6 @@ public synchronized void close() { * * @return containerLogDir. */ - public String getContainerLogDir() { return this.containerLogDir; } @@ -112,7 +111,7 @@ public void setContainerLogFile(String containerLogFile) { } private static final long EVENT_SIZE = 100; - + public long getTotalLogFileSize() { return maxEvents * EVENT_SIZE; } From f11baca77f1a1f42e8ec083db49c92ff4e61c495 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sun, 1 Oct 2023 07:44:33 +0800 Subject: [PATCH 7/7] fix checkstyle --- .../org/apache/hadoop/yarn/ContainerRollingLogAppender.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java index f0e00fc194064..acb47d9dd002b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java @@ -31,8 +31,7 @@ */ @Public @Unstable -public class ContainerRollingLogAppender extends RollingFileAppender - implements Flushable { +public class ContainerRollingLogAppender extends RollingFileAppender implements Flushable { private String containerLogDir; private String containerLogFile;