001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.quotas.policies;
019
020import static org.mockito.Mockito.mock;
021import static org.mockito.Mockito.when;
022
023import java.util.ArrayList;
024import java.util.List;
025import org.apache.hadoop.fs.FileStatus;
026import org.apache.hadoop.fs.FileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.TableName;
030import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
031import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
032import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
033import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
034import org.apache.hadoop.hbase.regionserver.RegionServerServices;
035import org.apache.hadoop.hbase.testclassification.SmallTests;
036import org.junit.Before;
037import org.junit.ClassRule;
038import org.junit.Test;
039import org.junit.experimental.categories.Category;
040
041@Category(SmallTests.class)
042public class TestBulkLoadCheckingViolationPolicyEnforcement {
043
044  @ClassRule
045  public static final HBaseClassTestRule CLASS_RULE =
046      HBaseClassTestRule.forClass(TestBulkLoadCheckingViolationPolicyEnforcement.class);
047
048  FileSystem fs;
049  RegionServerServices rss;
050  TableName tableName;
051  SpaceViolationPolicyEnforcement policy;
052
053  @Before
054  public void setup() {
055    fs = mock(FileSystem.class);
056    rss = mock(RegionServerServices.class);
057    tableName = TableName.valueOf("foo");
058    policy = new DefaultViolationPolicyEnforcement();
059  }
060
061  @Test
062  public void testFilesUnderLimit() throws Exception {
063    final List<String> paths = new ArrayList<>();
064    final List<FileStatus> statuses = new ArrayList<>();
065    final long length = 100L * 1024L;
066    for (int i = 0; i < 5; i++) {
067      String path = "/" + i;
068      FileStatus status = mock(FileStatus.class);
069      when(fs.getFileStatus(new Path(path))).thenReturn(status);
070      when(status.getLen()).thenReturn(length);
071      when(status.isFile()).thenReturn(true);
072      paths.add(path);
073      statuses.add(status);
074    }
075
076    // Quota is not in violation now
077    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6);
078
079    policy.initialize(rss, tableName, snapshot);
080
081    policy.checkBulkLoad(fs, paths);
082  }
083
084  @Test(expected = IllegalArgumentException.class)
085  public void testFileIsNotAFile() throws Exception {
086    final List<String> paths = new ArrayList<>();
087    String path = "/1";
088    FileStatus status = mock(FileStatus.class);
089    when(fs.getFileStatus(new Path(path))).thenReturn(status);
090    when(status.getLen()).thenReturn(1000L);
091    when(status.isFile()).thenReturn(false);
092    paths.add(path);
093
094    // Quota is not in violation now
095    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE);
096
097    policy.initialize(rss, tableName, snapshot);
098
099    // If the file to bulk load isn't a file, this should throw an exception
100    policy.checkBulkLoad(fs, paths);
101  }
102
103  @Test(expected = SpaceLimitingException.class)
104  public void testOneFileInBatchOverLimit() throws Exception {
105    final List<String> paths = new ArrayList<>();
106    final List<FileStatus> statuses = new ArrayList<>();
107    final long length = 1000L * 1024L;
108    for (int i = 0; i < 5; i++) {
109      String path = "/" + i;
110      FileStatus status = mock(FileStatus.class);
111      when(fs.getFileStatus(new Path(path))).thenReturn(status);
112      when(status.getLen()).thenReturn(length);
113      when(status.isFile()).thenReturn(true);
114      paths.add(path);
115      statuses.add(status);
116    }
117
118    // Quota is not in violation now
119    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L);
120
121    policy.initialize(rss, tableName, snapshot);
122
123    policy.checkBulkLoad(fs, paths);
124  }
125
126  @Test(expected = SpaceLimitingException.class)
127  public void testSumOfFilesOverLimit() throws Exception {
128    final List<String> paths = new ArrayList<>();
129    final List<FileStatus> statuses = new ArrayList<>();
130    final long length = 1024L;
131    for (int i = 0; i < 5; i++) {
132      String path = "/" + i;
133      FileStatus status = mock(FileStatus.class);
134      when(fs.getFileStatus(new Path(path))).thenReturn(status);
135      when(status.getLen()).thenReturn(length);
136      when(status.isFile()).thenReturn(true);
137      paths.add(path);
138      statuses.add(status);
139    }
140
141    // Quota is not in violation now, but 5*1024 files would push us to violation
142    SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L);
143
144    policy.initialize(rss, tableName, snapshot);
145
146    policy.checkBulkLoad(fs, paths);
147  }
148}