001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.apache.hadoop.hbase.master.balancer.CandidateGeneratorTestUtil.createMockBalancerClusterState;
021import static org.junit.jupiter.api.Assertions.assertEquals;
022import static org.junit.jupiter.api.Assertions.assertTrue;
023import static org.mockito.Mockito.when;
024
025import java.util.ArrayDeque;
026import java.util.Arrays;
027import java.util.Deque;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Random;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.RegionMetrics;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.Size;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.testclassification.MasterTests;
040import org.apache.hadoop.hbase.testclassification.SmallTests;
041import org.junit.jupiter.api.Tag;
042import org.junit.jupiter.api.Test;
043import org.mockito.Mockito;
044
045@Tag(MasterTests.TAG)
046@Tag(SmallTests.TAG)
047public class TestStoreFileTableSkewCostFunction {
048
049  private static final TableName DEFAULT_TABLE = TableName.valueOf("testTable");
050  private static final Map<Long, Integer> REGION_TO_STORE_FILE_SIZE_MB = new HashMap<>();
051
052  /**
053   * Tests that a uniform store file distribution (single table) across servers results in zero
054   * cost.
055   */
056  @Test
057  public void testUniformDistribution() {
058    ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L);
059    ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L);
060    ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L);
061    ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L);
062
063    Map<ServerName, List<RegionInfo>> serverToRegions = new HashMap<>();
064    serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
065    serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
066    serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
067    serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
068
069    BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions);
070    DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState);
071
072    StoreFileTableSkewCostFunction costFunction =
073      new StoreFileTableSkewCostFunction(new Configuration());
074    costFunction.prepare(state);
075    double cost = costFunction.cost();
076
077    // Expect zero cost since all regions (from the same table) are balanced.
078    assertEquals(0.0, cost, 1e-6, "Uniform distribution should yield zero cost");
079  }
080
081  /**
082   * Tests that a skewed store file distribution (single table) results in a positive cost.
083   */
084  @Test
085  public void testSkewedDistribution() {
086    ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L);
087    ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L);
088    ServerName server3 = ServerName.valueOf("server3.example.org", 1234, 1L);
089    ServerName server4 = ServerName.valueOf("server4.example.org", 1234, 1L);
090
091    Map<ServerName, List<RegionInfo>> serverToRegions = new HashMap<>();
092    // Three servers get regions with 10 store files each,
093    // while one server gets regions with 30 store files each.
094    serverToRegions.put(server1, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
095    serverToRegions.put(server2, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
096    serverToRegions.put(server3, Arrays.asList(createMockRegionInfo(10), createMockRegionInfo(10)));
097    serverToRegions.put(server4, Arrays.asList(createMockRegionInfo(30), createMockRegionInfo(30)));
098
099    BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions);
100    DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState);
101
102    StoreFileTableSkewCostFunction costFunction =
103      new StoreFileTableSkewCostFunction(new Configuration());
104    costFunction.prepare(state);
105    double cost = costFunction.cost();
106
107    // Expect a positive cost because the distribution is skewed.
108    assertTrue(cost > 0.0, "Skewed distribution should yield a positive cost");
109  }
110
111  /**
112   * Tests that an empty cluster (no servers/regions) is handled gracefully.
113   */
114  @Test
115  public void testEmptyDistribution() {
116    Map<ServerName, List<RegionInfo>> serverToRegions = new HashMap<>();
117
118    BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions);
119    DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState);
120
121    StoreFileTableSkewCostFunction costFunction =
122      new StoreFileTableSkewCostFunction(new Configuration());
123    costFunction.prepare(state);
124    double cost = costFunction.cost();
125
126    // Expect zero cost when there is no load.
127    assertEquals(0.0, cost, 1e-6, "Empty distribution should yield zero cost");
128  }
129
130  /**
131   * Tests that having multiple tables results in a positive cost when each table's regions are not
132   * balanced across servers – even if the overall load per server is balanced.
133   */
134  @Test
135  public void testMultipleTablesDistribution() {
136    // Two servers.
137    ServerName server1 = ServerName.valueOf("server1.example.org", 1234, 1L);
138    ServerName server2 = ServerName.valueOf("server2.example.org", 1234, 1L);
139
140    // Define two tables.
141    TableName table1 = TableName.valueOf("testTable1");
142    TableName table2 = TableName.valueOf("testTable2");
143
144    // For table1, all regions are on server1.
145    // For table2, all regions are on server2.
146    Map<ServerName, List<RegionInfo>> serverToRegions = new HashMap<>();
147    serverToRegions.put(server1,
148      Arrays.asList(createMockRegionInfo(table1, 10), createMockRegionInfo(table1, 10)));
149    serverToRegions.put(server2,
150      Arrays.asList(createMockRegionInfo(table2, 10), createMockRegionInfo(table2, 10)));
151
152    // Although each server gets 20 MB overall, table1 and table2 are not balanced across servers.
153    BalancerClusterState clusterState = createMockBalancerClusterState(serverToRegions);
154    DummyBalancerClusterState state = new DummyBalancerClusterState(clusterState);
155
156    StoreFileTableSkewCostFunction costFunction =
157      new StoreFileTableSkewCostFunction(new Configuration());
158    costFunction.prepare(state);
159    double cost = costFunction.cost();
160
161    // Expect a positive cost because the skew is computed per table.
162    assertTrue(cost > 0.0, "Multiple table distribution should yield a positive cost");
163  }
164
165  /**
166   * Helper method to create a RegionInfo for the default table with the given store file size.
167   */
168  private static RegionInfo createMockRegionInfo(int storeFileSizeMb) {
169    return createMockRegionInfo(DEFAULT_TABLE, storeFileSizeMb);
170  }
171
172  /**
173   * Helper method to create a RegionInfo for a specified table with the given store file size.
174   */
175  private static RegionInfo createMockRegionInfo(TableName table, int storeFileSizeMb) {
176    long regionId = new Random().nextLong();
177    REGION_TO_STORE_FILE_SIZE_MB.put(regionId, storeFileSizeMb);
178    return RegionInfoBuilder.newBuilder(table).setStartKey(generateRandomByteArray(4))
179      .setEndKey(generateRandomByteArray(4)).setReplicaId(0).setRegionId(regionId).build();
180  }
181
182  private static byte[] generateRandomByteArray(int n) {
183    byte[] byteArray = new byte[n];
184    new Random().nextBytes(byteArray);
185    return byteArray;
186  }
187
188  /**
189   * A simplified BalancerClusterState which ensures we provide the intended test RegionMetrics data
190   * when balancing this cluster
191   */
192  private static class DummyBalancerClusterState extends BalancerClusterState {
193    private final RegionInfo[] testRegions;
194
195    DummyBalancerClusterState(BalancerClusterState bcs) {
196      super(bcs.clusterState, null, null, null, null);
197      this.testRegions = bcs.regions;
198    }
199
200    @Override
201    Deque<BalancerRegionLoad>[] getRegionLoads() {
202      @SuppressWarnings("unchecked")
203      Deque<BalancerRegionLoad>[] loads = new Deque[testRegions.length];
204      for (int i = 0; i < testRegions.length; i++) {
205        Deque<BalancerRegionLoad> dq = new ArrayDeque<>();
206        dq.add(new BalancerRegionLoad(createMockRegionMetrics(testRegions[i])) {
207        });
208        loads[i] = dq;
209      }
210      return loads;
211    }
212  }
213
214  /**
215   * Creates a mocked RegionMetrics for the given region.
216   */
217  private static RegionMetrics createMockRegionMetrics(RegionInfo regionInfo) {
218    RegionMetrics regionMetrics = Mockito.mock(RegionMetrics.class);
219
220    // Important
221    int storeFileSizeMb = REGION_TO_STORE_FILE_SIZE_MB.get(regionInfo.getRegionId());
222    when(regionMetrics.getRegionSizeMB()).thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE));
223    when(regionMetrics.getStoreFileSize())
224      .thenReturn(new Size(storeFileSizeMb, Size.Unit.MEGABYTE));
225
226    // Not important
227    when(regionMetrics.getReadRequestCount()).thenReturn(0L);
228    when(regionMetrics.getCpRequestCount()).thenReturn(0L);
229    when(regionMetrics.getWriteRequestCount()).thenReturn(0L);
230    when(regionMetrics.getMemStoreSize()).thenReturn(new Size(0, Size.Unit.MEGABYTE));
231    when(regionMetrics.getCurrentRegionCachedRatio()).thenReturn(0.0f);
232    return regionMetrics;
233  }
234}