Skip to content

HADOOP-19422. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-archive-logs. #7621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 17, 2025
Merged

HADOOP-19422. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-archive-logs. #7621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a4e80b
HADOOP-19422. Upgrade JUnit from 4 to 5 in hadoop-archive-logs.
slfan1989 Apr 16, 2025
a44b5fe
HADOOP-19422. Fix CheckStyle Issue.
slfan1989 Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 88 additions & 77 deletions ...ools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@

package org.apache.hadoop.tools;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
Expand All @@ -37,8 +42,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;

import java.io.File;
import java.io.IOException;
Expand All @@ -55,7 +60,8 @@ public class TestHadoopArchiveLogs {
new Random().nextBytes(DUMMY_DATA);
}

@Test(timeout = 10000)
@Test
@Timeout(value = 10)
public void testCheckFilesAndSeedApps() throws Exception {
Configuration conf = new Configuration();
HadoopArchiveLogs hal = new HadoopArchiveLogs(conf);
Expand All @@ -64,7 +70,7 @@ public void testCheckFilesAndSeedApps() throws Exception {
String suffix = "logs";
Path logDir = new Path(rootLogDir, new Path(USER, suffix));
fs.delete(logDir, true);
Assert.assertFalse(fs.exists(logDir));
assertFalse(fs.exists(logDir));
fs.mkdirs(logDir);

// no files found
Expand Down Expand Up @@ -96,15 +102,16 @@ public void testCheckFilesAndSeedApps() throws Exception {
createFile(fs, new Path(app5Path, "file1"), 2);
createFile(fs, new Path(app5Path, "file2"), 3);

Assert.assertEquals(0, hal.eligibleApplications.size());
assertEquals(0, hal.eligibleApplications.size());
hal.checkFilesAndSeedApps(fs, rootLogDir, suffix, new Path(rootLogDir,
"archive-logs-work"));
Assert.assertEquals(1, hal.eligibleApplications.size());
Assert.assertEquals(appId5.toString(),
assertEquals(1, hal.eligibleApplications.size());
assertEquals(appId5.toString(),
hal.eligibleApplications.iterator().next().getAppId());
}

@Test(timeout = 10000)
@Test
@Timeout(value = 10)
public void testCheckMaxEligible() throws Exception {
Configuration conf = new Configuration();
HadoopArchiveLogs.AppInfo app1 = new HadoopArchiveLogs.AppInfo(
Expand All @@ -129,46 +136,47 @@ public void testCheckMaxEligible() throws Exception {
ApplicationId.newInstance(CLUSTER_TIMESTAMP, 7).toString(), USER);
app7.setFinishTime(CLUSTER_TIMESTAMP);
HadoopArchiveLogs hal = new HadoopArchiveLogs(conf);
Assert.assertEquals(0, hal.eligibleApplications.size());
assertEquals(0, hal.eligibleApplications.size());
hal.eligibleApplications.add(app1);
hal.eligibleApplications.add(app2);
hal.eligibleApplications.add(app3);
hal.eligibleApplications.add(app4);
hal.eligibleApplications.add(app5);
hal.eligibleApplications.add(app6);
hal.eligibleApplications.add(app7);
Assert.assertEquals(7, hal.eligibleApplications.size());
assertEquals(7, hal.eligibleApplications.size());
hal.maxEligible = -1;
hal.checkMaxEligible();
Assert.assertEquals(7, hal.eligibleApplications.size());
assertEquals(7, hal.eligibleApplications.size());
hal.maxEligible = 6;
hal.checkMaxEligible();
Assert.assertEquals(6, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app5));
assertEquals(6, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app5));
hal.maxEligible = 5;
hal.checkMaxEligible();
Assert.assertEquals(5, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app4));
assertEquals(5, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app4));
hal.maxEligible = 4;
hal.checkMaxEligible();
Assert.assertEquals(4, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app7));
assertEquals(4, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app7));
hal.maxEligible = 3;
hal.checkMaxEligible();
Assert.assertEquals(3, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app1));
assertEquals(3, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app1));
hal.maxEligible = 2;
hal.checkMaxEligible();
Assert.assertEquals(2, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app2));
assertEquals(2, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app2));
hal.maxEligible = 1;
hal.checkMaxEligible();
Assert.assertEquals(1, hal.eligibleApplications.size());
Assert.assertFalse(hal.eligibleApplications.contains(app6));
Assert.assertTrue(hal.eligibleApplications.contains(app3));
assertEquals(1, hal.eligibleApplications.size());
assertFalse(hal.eligibleApplications.contains(app6));
assertTrue(hal.eligibleApplications.contains(app3));
}

@Test(timeout = 30000)
@Test
@Timeout(value = 30)
public void testFilterAppsByAggregatedStatus() throws Exception {
try (MiniYARNCluster yarnCluster =
new MiniYARNCluster(TestHadoopArchiveLogs.class.getSimpleName(),
Expand Down Expand Up @@ -206,7 +214,7 @@ public void testFilterAppsByAggregatedStatus() throws Exception {
// appImpl8 is not in the RM

HadoopArchiveLogs hal = new HadoopArchiveLogs(conf);
Assert.assertEquals(0, hal.eligibleApplications.size());
assertEquals(0, hal.eligibleApplications.size());
hal.eligibleApplications.add(
new HadoopArchiveLogs.AppInfo(appImpl1.getApplicationId().toString(),
USER));
Expand Down Expand Up @@ -234,16 +242,17 @@ public void testFilterAppsByAggregatedStatus() throws Exception {
new HadoopArchiveLogs.AppInfo(appImpl8.getApplicationId().toString(),
USER);
hal.eligibleApplications.add(app8);
Assert.assertEquals(8, hal.eligibleApplications.size());
assertEquals(8, hal.eligibleApplications.size());
hal.filterAppsByAggregatedStatus();
Assert.assertEquals(3, hal.eligibleApplications.size());
Assert.assertTrue(hal.eligibleApplications.contains(app4));
Assert.assertTrue(hal.eligibleApplications.contains(app7));
Assert.assertTrue(hal.eligibleApplications.contains(app8));
assertEquals(3, hal.eligibleApplications.size());
assertTrue(hal.eligibleApplications.contains(app4));
assertTrue(hal.eligibleApplications.contains(app7));
assertTrue(hal.eligibleApplications.contains(app8));
}
}

@Test(timeout = 10000)
@Test
@Timeout(value = 10)
public void testGenerateScript() throws Exception {
_testGenerateScript(false);
_testGenerateScript(true);
Expand Down Expand Up @@ -276,59 +285,59 @@ private void _testGenerateScript(boolean proxy) throws Exception {

File localScript = new File("target", "script.sh");
localScript.delete();
Assert.assertFalse(localScript.exists());
assertFalse(localScript.exists());
hal.generateScript(localScript);
Assert.assertTrue(localScript.exists());
assertTrue(localScript.exists());
String script = IOUtils.toString(localScript.toURI(), StandardCharsets.UTF_8);
String[] lines = script.split("\n");
Assert.assertEquals(22, lines.length);
Assert.assertEquals("#!/bin/bash", lines[0]);
Assert.assertEquals("set -e", lines[1]);
Assert.assertEquals("set -x", lines[2]);
Assert.assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]);
assertEquals(22, lines.length);
assertEquals("#!/bin/bash", lines[0]);
assertEquals("set -e", lines[1]);
assertEquals("set -x", lines[2]);
assertEquals("if [ \"$YARN_SHELL_ID\" == \"1\" ]; then", lines[3]);
boolean oneBefore = true;
if (lines[4].contains(app1.toString())) {
Assert.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]);
Assert.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]);
assertEquals("\tappId=\"" + app1.toString() + "\"", lines[4]);
assertEquals("\tappId=\"" + app2.toString() + "\"", lines[10]);
} else {
oneBefore = false;
Assert.assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]);
Assert.assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]);
assertEquals("\tappId=\"" + app2.toString() + "\"", lines[4]);
assertEquals("\tappId=\"" + app1.toString() + "\"", lines[10]);
}
Assert.assertEquals("\tuser=\"" + USER + "\"", lines[5]);
Assert.assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString()
assertEquals("\tuser=\"" + USER + "\"", lines[5]);
assertEquals("\tworkingDir=\"" + (oneBefore ? workingDir.toString()
: workingDir2.toString()) + "\"", lines[6]);
Assert.assertEquals("\tremoteRootLogDir=\"" + (oneBefore
assertEquals("\tremoteRootLogDir=\"" + (oneBefore
? remoteRootLogDir.toString() : remoteRootLogDir2.toString())
+ "\"", lines[7]);
Assert.assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2)
assertEquals("\tsuffix=\"" + (oneBefore ? suffix : suffix2)
+ "\"", lines[8]);
Assert.assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then",
assertEquals("elif [ \"$YARN_SHELL_ID\" == \"2\" ]; then",
lines[9]);
Assert.assertEquals("\tuser=\"" + USER + "\"", lines[11]);
Assert.assertEquals("\tworkingDir=\"" + (oneBefore
assertEquals("\tuser=\"" + USER + "\"", lines[11]);
assertEquals("\tworkingDir=\"" + (oneBefore
? workingDir2.toString() : workingDir.toString()) + "\"",
lines[12]);
Assert.assertEquals("\tremoteRootLogDir=\"" + (oneBefore
assertEquals("\tremoteRootLogDir=\"" + (oneBefore
? remoteRootLogDir2.toString() : remoteRootLogDir.toString())
+ "\"", lines[13]);
Assert.assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix)
assertEquals("\tsuffix=\"" + (oneBefore ? suffix2 : suffix)
+ "\"", lines[14]);
Assert.assertEquals("else", lines[15]);
Assert.assertEquals("\techo \"Unknown Mapping!\"", lines[16]);
Assert.assertEquals("\texit 1", lines[17]);
Assert.assertEquals("fi", lines[18]);
Assert.assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]);
Assert.assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH="));
assertEquals("else", lines[15]);
assertEquals("\techo \"Unknown Mapping!\"", lines[16]);
assertEquals("\texit 1", lines[17]);
assertEquals("fi", lines[18]);
assertEquals("export HADOOP_CLIENT_OPTS=\"-Xmx1024m\"", lines[19]);
assertTrue(lines[20].startsWith("export HADOOP_CLASSPATH="));
if (proxy) {
Assert.assertEquals(
assertEquals(
"\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." +
"HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " +
"-workingDir \"$workingDir\" -remoteRootLogDir " +
"\"$remoteRootLogDir\" -suffix \"$suffix\"",
lines[21]);
} else {
Assert.assertEquals(
assertEquals(
"\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." +
"HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " +
"-workingDir \"$workingDir\" -remoteRootLogDir " +
Expand All @@ -343,7 +352,8 @@ private void _testGenerateScript(boolean proxy) throws Exception {
* are updated as well, if necessary.
* @throws Exception
*/
@Test(timeout = 5000)
@Test
@Timeout(value = 5)
public void testStatuses() throws Exception {
LogAggregationStatus[] statuses = new LogAggregationStatus[7];
statuses[0] = LogAggregationStatus.DISABLED;
Expand All @@ -353,51 +363,52 @@ public void testStatuses() throws Exception {
statuses[4] = LogAggregationStatus.SUCCEEDED;
statuses[5] = LogAggregationStatus.FAILED;
statuses[6] = LogAggregationStatus.TIME_OUT;
Assert.assertArrayEquals(statuses, LogAggregationStatus.values());
assertArrayEquals(statuses, LogAggregationStatus.values());
}

@Test(timeout = 5000)
@Test
@Timeout(value = 5)
public void testPrepareWorkingDir() throws Exception {
Configuration conf = new Configuration();
HadoopArchiveLogs hal = new HadoopArchiveLogs(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path workingDir = new Path("target", "testPrepareWorkingDir");
fs.delete(workingDir, true);
Assert.assertFalse(fs.exists(workingDir));
assertFalse(fs.exists(workingDir));
// -force is false and the dir doesn't exist so it will create one
hal.force = false;
boolean dirPrepared = hal.prepareWorkingDir(fs, workingDir);
Assert.assertTrue(dirPrepared);
Assert.assertTrue(fs.exists(workingDir));
Assert.assertEquals(
assertTrue(dirPrepared);
assertTrue(fs.exists(workingDir));
assertEquals(
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
// Throw a file in the dir
Path dummyFile = new Path(workingDir, "dummy.txt");
fs.createNewFile(dummyFile);
Assert.assertTrue(fs.exists(dummyFile));
assertTrue(fs.exists(dummyFile));
// -force is false and the dir exists, so nothing will happen and the dummy
// still exists
dirPrepared = hal.prepareWorkingDir(fs, workingDir);
Assert.assertFalse(dirPrepared);
Assert.assertTrue(fs.exists(workingDir));
Assert.assertTrue(fs.exists(dummyFile));
Assert.assertEquals(
assertFalse(dirPrepared);
assertTrue(fs.exists(workingDir));
assertTrue(fs.exists(dummyFile));
assertEquals(
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
// -force is true and the dir exists, so it will recreate it and the dummy
// won't exist anymore
hal.force = true;
dirPrepared = hal.prepareWorkingDir(fs, workingDir);
Assert.assertTrue(dirPrepared);
Assert.assertTrue(fs.exists(workingDir));
Assert.assertEquals(
assertTrue(dirPrepared);
assertTrue(fs.exists(workingDir));
assertEquals(
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
Assert.assertFalse(fs.exists(dummyFile));
assertFalse(fs.exists(dummyFile));
}

private static void createFile(FileSystem fs, Path p, long sizeMultiple)
Expand All @@ -413,7 +424,7 @@ private static void createFile(FileSystem fs, Path p, long sizeMultiple)
out.close();
}
}
Assert.assertTrue(fs.exists(p));
assertTrue(fs.exists(p));
}

private static RMApp createRMApp(int id, Configuration conf, RMContext rmContext,
Expand Down
Loading