001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup.replication;
019
020import java.io.IOException;
021import java.util.concurrent.atomic.AtomicLong;
022import org.apache.hadoop.fs.FSDataOutputStream;
023import org.apache.hadoop.fs.FileSystem;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor;
026import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
027import org.apache.hadoop.hbase.util.AtomicUtils;
028import org.apache.hadoop.hbase.util.CommonFSUtils;
029import org.apache.yetus.audience.InterfaceAudience;
030
031/**
032 * A custom implementation of {@link ProtobufLogWriter} that provides support for writing
033 * protobuf-based WAL (Write-Ahead Log) entries to object store-backed files.
034 * <p>
035 * This class overrides the {@link ProtobufLogWriter#sync(boolean)} and
036 * {@link ProtobufLogWriter#initOutput(FileSystem, Path, boolean, int, short, long, StreamSlowMonitor, boolean)}
037 * methods to ensure compatibility with object stores, while ignoring specific capability checks
038 * such as HFLUSH and HSYNC. These checks are often not supported by some object stores, and
039 * bypassing them ensures smooth operation in such environments.
040 * </p>
041 */
042@InterfaceAudience.Private
043public class ObjectStoreProtobufWalWriter extends ProtobufLogWriter {
044  private final AtomicLong syncedLength = new AtomicLong(0);
045
046  @Override
047  public void sync(boolean forceSync) throws IOException {
048    FSDataOutputStream fsDataOutputstream = this.output;
049    if (fsDataOutputstream == null) {
050      return; // Presume closed
051    }
052    // Special case for Hadoop S3: Unlike traditional file systems, where flush() ensures data is
053    // durably written, in Hadoop S3, flush() only writes data to the internal buffer and does not
054    // immediately persist it to S3. The actual upload to S3 happens asynchronously, typically when
055    // a block is full or when close() is called, which finalizes the upload process.
056    fsDataOutputstream.flush();
057    AtomicUtils.updateMax(this.syncedLength, fsDataOutputstream.getPos());
058  }
059
060  @Override
061  protected void initOutput(FileSystem fs, Path path, boolean overwritable, int bufferSize,
062    short replication, long blockSize, StreamSlowMonitor monitor, boolean noLocalWrite)
063    throws IOException {
064    try {
065      super.initOutput(fs, path, overwritable, bufferSize, replication, blockSize, monitor,
066        noLocalWrite);
067    } catch (CommonFSUtils.StreamLacksCapabilityException e) {
068      // Ignore capability check for HFLUSH and HSYNC capabilities
069      // Some object stores may not support these capabilities, so we bypass the exception handling
070      // to ensure compatibility with such stores.
071    }
072  }
073}