View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import java.io.IOException;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Set;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.HRegionInfo;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.catalog.MetaReader.Visitor;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.master.MasterServices;
34  import org.apache.hadoop.hbase.migration.HRegionInfo090x;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.Writables;
37  
38  /**
39   * Tools to help with migration of meta tables so they no longer host
40   * instances of HTableDescriptor.
41   * @deprecated Used migration from 0.90 to 0.92 so will be going away in next
42   * release
43   */
44  public class MetaMigrationRemovingHTD {
45    private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class);
46  
47    /**
48     * Update legacy META rows, removing HTD from HRI.
49     * @param masterServices
50     * @return List of table descriptors.
51     * @throws IOException
52     */
53    public static Set<HTableDescriptor> updateMetaWithNewRegionInfo(
54        final MasterServices masterServices)
55    throws IOException {
56      MigratingVisitor v = new MigratingVisitor(masterServices);
57      MetaReader.fullScan(masterServices.getCatalogTracker(), v);
58      updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker());
59      return v.htds;
60    }
61  
62    /**
63     * Update the ROOT with new HRI. (HRI with no HTD)
64     * @param masterServices
65     * @return List of table descriptors
66     * @throws IOException
67     */
68    static Set<HTableDescriptor> updateRootWithNewRegionInfo(
69        final MasterServices masterServices)
70    throws IOException {
71      MigratingVisitor v = new MigratingVisitor(masterServices);
72      MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
73      return v.htds;
74    }
75  
76    /**
77     * Meta visitor that migrates the info:regioninfo as it visits.
78     */
79    static class MigratingVisitor implements Visitor {
80      private final MasterServices services;
81      final Set<HTableDescriptor> htds = new HashSet<HTableDescriptor>();
82  
83      MigratingVisitor(final MasterServices services) {
84        this.services = services;
85      }
86  
87      @Override
88      public boolean visit(Result r) throws IOException {
89        if (r ==  null || r.isEmpty()) return true;
90        // Check info:regioninfo, info:splitA, and info:splitB.  Make sure all
91        // have migrated HRegionInfos... that there are no leftover 090 version
92        // HRegionInfos.
93        byte [] hriBytes = getBytes(r, HConstants.REGIONINFO_QUALIFIER);
94        // Presumes that an edit updating all three cells either succeeds or
95        // doesn't -- that we don't have case of info:regioninfo migrated but not
96        // info:splitA.
97        if (isMigrated(hriBytes)) return true;
98        // OK. Need to migrate this row in meta.
99        HRegionInfo090x hri090 = getHRegionInfo090x(hriBytes);
100       HTableDescriptor htd = hri090.getTableDesc();
101       if (htd == null) {
102         LOG.warn("A 090 HRI has null HTD? Continuing; " + hri090.toString());
103         return true;
104       }
105       if (!this.htds.contains(htd)) {
106         // If first time we are adding a table, then write it out to fs.
107         // Presumes that first region in table has THE table's schema which
108         // might not be too bad of a presumption since it'll be first region
109         // 'altered'
110         this.services.getMasterFileSystem().createTableDescriptor(htd);
111         this.htds.add(htd);
112       }
113       // This will 'migrate' the hregioninfo from 090 version to 092.
114       HRegionInfo hri = new HRegionInfo(hri090);
115       // Now make a put to write back to meta.
116       Put p = new Put(hri.getRegionName());
117       p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
118         Writables.getBytes(hri));
119       // Now check info:splitA and info:splitB if present.  Migrate these too.
120       checkSplit(r, p, HConstants.SPLITA_QUALIFIER);
121       checkSplit(r, p, HConstants.SPLITB_QUALIFIER);
122       // Below we fake out putToCatalogTable
123       MetaEditor.putToCatalogTable(this.services.getCatalogTracker(), p);
124       LOG.info("Migrated " + Bytes.toString(p.getRow()));
125       return true;
126     }
127   }
128 
129   static void checkSplit(final Result r, final Put p, final byte [] which)
130   throws IOException {
131     byte [] hriSplitBytes = getBytes(r, which);
132     if (!isMigrated(hriSplitBytes)) {
133       // This will convert the HRI from 090 to 092 HRI.
134       HRegionInfo hri = Writables.getHRegionInfo(hriSplitBytes);
135       p.add(HConstants.CATALOG_FAMILY, which, Writables.getBytes(hri));
136     }
137   }
138 
139   /**
140    * @param r Result to dig in.
141    * @param qualifier Qualifier to look at in the passed <code>r</code>.
142    * @return Bytes for an HRegionInfo or null if no bytes or empty bytes found.
143    */
144   static byte [] getBytes(final Result r, final byte [] qualifier) {
145     byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
146     if (hriBytes == null || hriBytes.length <= 0) return null;
147     return hriBytes;
148   }
149 
150   /**
151    * @param r Result to look in.
152    * @param qualifier What to look at in the passed result.
153    * @return Either a 090 vintage HRegionInfo OR null if no HRegionInfo or
154    * the HRegionInfo is up to date and not in need of migration.
155    * @throws IOException
156    */
157   static HRegionInfo090x get090HRI(final Result r, final byte [] qualifier)
158   throws IOException {
159     byte [] hriBytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
160     if (hriBytes == null || hriBytes.length <= 0) return null;
161     if (isMigrated(hriBytes)) return null;
162     return getHRegionInfo090x(hriBytes);
163   }
164 
165   static boolean isMigrated(final byte [] hriBytes) {
166     if (hriBytes == null || hriBytes.length <= 0) return true;
167     // Else, what version this HRegionInfo instance is at.  The first byte
168     // is the version byte in a serialized HRegionInfo.  If its same as our
169     // current HRI, then nothing to do.
170     if (hriBytes[0] == HRegionInfo.VERSION) return true;
171     if (hriBytes[0] == HRegionInfo.VERSION_PRE_092) return false;
172     // Unknown version.  Return true that its 'migrated' but log warning.
173     // Should 'never' happen.
174     assert false: "Unexpected version; bytes=" + Bytes.toStringBinary(hriBytes);
175     return true;
176   }
177 
178   /**
179    * Migrate root and meta to newer version. This updates the META and ROOT
180    * and removes the HTD from HRI.
181    * @param masterServices
182    * @throws IOException
183    */
184   public static void migrateRootAndMeta(final MasterServices masterServices)
185       throws IOException {
186     updateRootWithNewRegionInfo(masterServices);
187     updateMetaWithNewRegionInfo(masterServices);
188   }
189 
190   /**
191    * Update the version flag in -ROOT-.
192    * @param catalogTracker
193    * @throws IOException
194    */
195   public static void updateRootWithMetaMigrationStatus(final CatalogTracker catalogTracker)
196   throws IOException {
197     Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
198     MetaEditor.putToRootTable(catalogTracker, setMetaVersion(p));
199     LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION);
200   }
201 
202   static Put setMetaVersion(final Put p) {
203     p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER,
204       Bytes.toBytes(HConstants.META_VERSION));
205     return p;
206   }
207 
208   /**
209    * @return True if the meta table has been migrated.
210    * @throws IOException
211    */
212   // Public because used in tests
213   public static boolean isMetaHRIUpdated(final MasterServices services)
214       throws IOException {
215     List<Result> results = MetaReader.fullScanOfRoot(services.getCatalogTracker());
216     if (results == null || results.isEmpty()) {
217       LOG.info("Not migrated");
218       return false;
219     }
220     // Presume only the one result because we only support on meta region.
221     Result r = results.get(0);
222     short version = getMetaVersion(r);
223     boolean migrated = version >= HConstants.META_VERSION;
224     LOG.info("Meta version=" + version + "; migrated=" + migrated);
225     return migrated;
226   }
227 
228   /**
229    * @param r Result to look at
230    * @return Current meta table version or -1 if no version found.
231    */
232   static short getMetaVersion(final Result r) {
233     byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
234         HConstants.META_VERSION_QUALIFIER);
235     return value == null || value.length <= 0? -1: Bytes.toShort(value);
236   }
237 
238   /**
239    * @return True if migrated.
240    * @throws IOException
241    */
242   public static boolean updateMetaWithNewHRI(final MasterServices services)
243   throws IOException {
244     if (isMetaHRIUpdated(services)) {
245       LOG.info("ROOT/Meta already up-to date with new HRI.");
246       return true;
247     }
248     LOG.info("Meta has HRI with HTDs. Updating meta now.");
249     try {
250       migrateRootAndMeta(services);
251       LOG.info("ROOT and Meta updated with new HRI.");
252       return true;
253     } catch (IOException e) {
254       throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
255         "Master startup aborted.");
256     }
257   }
258 
259   /**
260    * Get HREgionInfoForMigration serialized from bytes.
261    * @param bytes serialized bytes
262    * @return An instance of a 090 HRI or null if we failed deserialize
263    */
264   public static HRegionInfo090x getHRegionInfo090x(final byte [] bytes) {
265     if (bytes == null || bytes.length == 0) return null;
266     HRegionInfo090x hri = null;
267     try {
268       hri = (HRegionInfo090x)Writables.getWritable(bytes, new HRegionInfo090x());
269     } catch (IOException ioe) {
270       LOG.warn("Failed deserialize as a 090 HRegionInfo); bytes=" +
271         Bytes.toStringBinary(bytes), ioe);
272     }
273     return hri;
274   }
275 }