/*
* Copyright 2017 LinkedIn Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*
*/
package azkaban.execapp;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import azkaban.execapp.FlowPreparer.ProjectsDirCacheMetrics;
import azkaban.executor.ExecutableFlow;
import azkaban.project.ProjectFileHandler;
import azkaban.storage.StorageManager;
import azkaban.utils.FileIOUtils;
import azkaban.utils.Pair;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class FlowPreparerTest {
public static final String SAMPLE_FLOW_01 = "sample_flow_01";
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
private File executionsDir;
private File projectsDir;
private FlowPreparer instance;
private StorageManager createMockStorageManager() {
final ClassLoader classLoader = getClass().getClassLoader();
final File file = new File(classLoader.getResource(SAMPLE_FLOW_01 + ".zip").getFile());
final ProjectFileHandler projectFileHandler = mock(ProjectFileHandler.class);
when(projectFileHandler.getFileType()).thenReturn("zip");
when(projectFileHandler.getLocalFile()).thenReturn(file);
final StorageManager storageManager = mock(StorageManager.class);
when(storageManager.getProjectFile(anyInt(), anyInt())).thenReturn(projectFileHandler);
return storageManager;
}
@Before
public void setUp() throws Exception {
this.executionsDir = this.temporaryFolder.newFolder("executions");
this.projectsDir = this.temporaryFolder.newFolder("projects");
this.instance = spy(
new FlowPreparer(createMockStorageManager(), this.executionsDir, this.projectsDir, null));
doNothing().when(this.instance).touchIfExists(any());
}
@Test
public void testSetupProject() throws Exception {
final ProjectVersion pv = new ProjectVersion(12, 34,
new File(this.projectsDir, "sample_project_01"));
this.instance.setupProject(pv);
final long actualDirSize = 1048835;
assertThat(pv.getDirSizeInBytes()).isEqualTo(actualDirSize);
assertThat(FileIOUtils.readNumberFromFile(
Paths.get(pv.getInstalledDir().getPath(), FlowPreparer.PROJECT_DIR_SIZE_FILE_NAME)))
.isEqualTo(actualDirSize);
assertThat(FileIOUtils.readNumberFromFile(
Paths.get(pv.getInstalledDir().getPath(), FlowPreparer.PROJECT_DIR_COUNT_FILE_NAME)))
.isEqualTo(8);
assertTrue(pv.getInstalledDir().exists());
assertTrue(new File(pv.getInstalledDir(), "sample_flow_01").exists());
}
@Test
public void testSetupProjectTouchesTheDirSizeFile() throws Exception {
//verifies setup project touches project dir size file.
final ProjectVersion pv = new ProjectVersion(12, 34,
new File(this.projectsDir, "sample_project_01"));
//setup project 1st time will not do touch
this.instance.setupProject(pv);
verify(this.instance, never()).touchIfExists(
Paths.get(pv.getInstalledDir().getPath(), FlowPreparer.PROJECT_DIR_SIZE_FILE_NAME));
this.instance.setupProject(pv);
verify(this.instance).touchIfExists(
Paths.get(pv.getInstalledDir().getPath(), FlowPreparer.PROJECT_DIR_SIZE_FILE_NAME));
}
@Test
public void testSetupFlow() {
final ExecutableFlow executableFlow = mock(ExecutableFlow.class);
when(executableFlow.getExecutionId()).thenReturn(12345);
when(executableFlow.getProjectId()).thenReturn(12);
when(executableFlow.getVersion()).thenReturn(34);
this.instance.setup(executableFlow);
final File execDir = new File(this.executionsDir, "12345");
assertTrue(execDir.exists());
assertTrue(new File(execDir, SAMPLE_FLOW_01).exists());
}
@Test
public void testFileCountCheckNotCalled() {
//given
final ExecutableFlow executableFlow = mock(ExecutableFlow.class);
when(executableFlow.getExecutionId()).thenReturn(12345);
when(executableFlow.getProjectId()).thenReturn(12);
when(executableFlow.getVersion()).thenReturn(34);
//when
this.instance.setup(executableFlow);
//then
verify(this.instance, never()).isFileCountEqual(any(), anyInt());
}
@Test
public void testIsFileCountEqual() {
//given
final FlowPreparer flowPreparer = new FlowPreparer(createMockStorageManager(),
this.executionsDir, this.projectsDir, 1L);
final File projectDir = new File(this.projectsDir, "sample_project_01");
projectDir.mkdir();
final ProjectVersion pv = new ProjectVersion(1, 1, projectDir);
//then
assertThat(flowPreparer.isFileCountEqual(pv, 1)).isEqualTo(true);
}
@Test
public void testProjectCacheDirCleanerNotEnabled() throws IOException {
final Map<Pair<Integer, Integer>, ProjectVersion> installedProjects = new HashMap<>();
//given
final FlowPreparer flowPreparer = new FlowPreparer(createMockStorageManager(),
this.executionsDir, this.projectsDir, null);
//when
final List<File> expectedRemainingFiles = new ArrayList<>();
for (int i = 1; i <= 3; i++) {
final int projectId = i;
final int version = 1;
final ProjectVersion pv = new ProjectVersion(projectId, version, null);
installedProjects.put(new Pair<>(projectId, version), pv);
flowPreparer.setupProject(pv);
expectedRemainingFiles.add(pv.getInstalledDir());
}
//then
assertThat(this.projectsDir.listFiles()).containsExactlyInAnyOrder(expectedRemainingFiles
.toArray(new File[expectedRemainingFiles.size()]));
}
@Test
public void testProjectCacheDirCleaner() throws IOException, InterruptedException {
final Long projectDirMaxSize = 3L;
//given
final FlowPreparer flowPreparer = new FlowPreparer(createMockStorageManager(),
this.executionsDir, this.projectsDir, projectDirMaxSize);
//when
final List<File> expectedRemainingFiles = new ArrayList<>();
for (int i = 1; i <= 3; i++) {
final int projectId = i;
final int version = 1;
final ProjectVersion pv = new ProjectVersion(projectId, version, null);
flowPreparer.setupProject(pv);
if (i >= 2) {
//the first file will be deleted
expectedRemainingFiles.add(pv.getInstalledDir());
}
// last modified time of millis second granularity of a file is not supported by all file
// systems, so sleep for 1 second between creation of each project dir to make their last
// modified time different.
Thread.sleep(1000);
}
//then
assertThat(this.projectsDir.listFiles()).containsExactlyInAnyOrder(expectedRemainingFiles
.toArray(new File[expectedRemainingFiles.size()]));
}
@Test
public void testProjectsCacheMetricsZeroHit() {
//given
final FlowPreparer.ProjectsDirCacheMetrics cacheMetrics = new ProjectsDirCacheMetrics();
//when zero hit and zero miss then
assertThat(cacheMetrics.getHitRatio()).isEqualTo(0);
//when
cacheMetrics.incrementCacheMiss();
//then
assertThat(cacheMetrics.getHitRatio()).isEqualTo(0);
}
@Test
public void testProjectsCacheMetricsHit() {
//given
final FlowPreparer.ProjectsDirCacheMetrics cacheMetrics = new ProjectsDirCacheMetrics();
//when one hit
cacheMetrics.incrementCacheHit();
//then
assertThat(cacheMetrics.getHitRatio()).isEqualTo(1);
//when one miss
cacheMetrics.incrementCacheMiss();
//then
assertThat(cacheMetrics.getHitRatio()).isEqualTo(0.5);
}
}