From b55d958906b01d04db4595fb9497f68ca3efec9a Mon Sep 17 00:00:00 2001 From: chicm Date: Mon, 24 Nov 2014 10:57:51 +0800 Subject: [PATCH] Revert "update" --- bin/hbase.cmd | 4 +- dev-support/make_patch.sh | 142 ---- dev-support/rebase_all_git_branches.sh | 204 ----- dev-support/test-patch.sh | 10 +- .../src/main/assembly/components.xml | 8 - hbase-checkstyle/pom.xml | 43 +- .../hbase/checkstyle-suppressions.xml | 30 +- .../src/main/resources/hbase/checkstyle.xml | 30 +- hbase-client/pom.xml | 19 - .../java/org/apache/hadoop/hbase/Chore.java | 10 +- .../org/apache/hadoop/hbase/ClusterId.java | 5 +- .../apache/hadoop/hbase/ClusterStatus.java | 51 +- .../hadoop/hbase/CoprocessorEnvironment.java | 2 +- .../hadoop/hbase/DoNotRetryIOException.java | 1 + .../hbase/DroppedSnapshotException.java | 4 +- .../hadoop/hbase/HColumnDescriptor.java | 105 +-- .../org/apache/hadoop/hbase/HRegionInfo.java | 11 +- .../apache/hadoop/hbase/HRegionLocation.java | 3 +- .../apache/hadoop/hbase/HTableDescriptor.java | 32 +- .../InvalidFamilyOperationException.java | 2 + .../apache/hadoop/hbase/KeepDeletedCells.java | 45 -- .../hbase/MasterNotRunningException.java | 4 +- .../hadoop/hbase/MetaTableAccessor.java | 28 +- .../hbase/NotServingRegionException.java | 4 +- .../hadoop/hbase/PleaseHoldException.java | 1 + .../apache/hadoop/hbase/RegionException.java | 1 + .../org/apache/hadoop/hbase/RegionLoad.java | 11 - .../org/apache/hadoop/hbase/ServerLoad.java | 13 +- .../hadoop/hbase/TableExistsException.java | 1 + .../hbase/TableNotDisabledException.java | 1 + .../hbase/TableNotEnabledException.java | 1 + .../hadoop/hbase/TableNotFoundException.java | 1 + .../hadoop/hbase/YouAreDeadException.java | 4 +- .../hbase/ZooKeeperConnectionException.java | 4 +- .../apache/hadoop/hbase/client/Action.java | 3 +- .../org/apache/hadoop/hbase/client/Admin.java | 15 +- .../apache/hadoop/hbase/client/Append.java | 4 +- .../hadoop/hbase/client/Attributes.java | 4 +- .../hbase/client/ClientIdGenerator.java | 5 + .../hadoop/hbase/client/ClientScanner.java | 3 +- .../client/ClientSmallReversedScanner.java | 8 +- .../hbase/client/ClientSmallScanner.java | 2 +- .../hbase/client/ClusterConnection.java | 13 +- .../hbase/client/ClusterStatusListener.java | 25 +- .../hadoop/hbase/client/Connection.java | 4 +- .../hbase/client/ConnectionAdapter.java | 7 +- .../hbase/client/ConnectionFactory.java | 4 +- .../hbase/client/ConnectionManager.java | 12 +- .../hadoop/hbase/client/ConnectionUtils.java | 2 +- .../apache/hadoop/hbase/client/Delete.java | 113 +-- .../hadoop/hbase/client/FailureInfo.java | 61 -- .../client/FastFailInterceptorContext.java | 123 --- .../org/apache/hadoop/hbase/client/Get.java | 2 +- .../hadoop/hbase/client/HBaseAdmin.java | 71 +- .../hadoop/hbase/client/HConnectable.java | 2 +- .../hadoop/hbase/client/HConnection.java | 4 +- .../hadoop/hbase/client/HConnectionKey.java | 4 +- .../hbase/client/HConnectionManager.java | 5 +- .../apache/hadoop/hbase/client/HTable.java | 6 +- .../hadoop/hbase/client/HTableFactory.java | 6 +- .../hbase/client/HTableInterfaceFactory.java | 6 +- .../apache/hadoop/hbase/client/Increment.java | 8 +- .../apache/hadoop/hbase/client/MetaCache.java | 3 +- .../hadoop/hbase/client/MetaScanner.java | 2 +- .../hadoop/hbase/client/MultiAction.java | 2 +- .../hadoop/hbase/client/MultiResponse.java | 4 + .../apache/hadoop/hbase/client/Mutation.java | 4 +- .../NoOpRetryableCallerInterceptor.java | 68 -- .../NoOpRetryingInterceptorContext.java | 44 -- .../client/NoServerForRegionException.java | 2 +- .../hbase/client/OperationWithAttributes.java | 8 +- .../client/PerClientRandomNonceGenerator.java | 2 +- .../client/PreemptiveFastFailInterceptor.java | 405 ---------- .../org/apache/hadoop/hbase/client/Put.java | 4 +- .../org/apache/hadoop/hbase/client/Query.java | 1 - .../client/RegionCoprocessorServiceExec.java | 7 +- .../hadoop/hbase/client/RegionLocator.java | 12 +- .../hbase/client/RegionOfflineException.java | 2 +- .../hbase/client/RegionReplicaUtil.java | 2 +- .../hbase/client/RegionServerCallable.java | 4 +- .../apache/hadoop/hbase/client/Registry.java | 1 + .../apache/hadoop/hbase/client/Result.java | 4 +- .../hadoop/hbase/client/ResultScanner.java | 6 +- .../client/RetriesExhaustedException.java | 7 +- .../RetriesExhaustedWithDetailsException.java | 10 +- .../client/RetryingCallerInterceptor.java | 98 --- .../RetryingCallerInterceptorContext.java | 69 -- .../RetryingCallerInterceptorFactory.java | 81 -- .../hbase/client/ReversedClientScanner.java | 2 +- .../hbase/client/ReversedScannerCallable.java | 4 +- .../hbase/client/RowTooBigException.java | 2 +- .../hbase/client/RpcRetryingCaller.java | 23 +- .../client/RpcRetryingCallerFactory.java | 15 +- .../RpcRetryingCallerWithReadReplicas.java | 30 +- .../org/apache/hadoop/hbase/client/Scan.java | 3 +- .../hadoop/hbase/client/ScannerCallable.java | 4 +- .../client/ScannerCallableWithReplicas.java | 2 +- .../hbase/client/ScannerTimeoutException.java | 2 +- .../org/apache/hadoop/hbase/client/Table.java | 14 +- .../hadoop/hbase/client/TableState.java | 2 +- .../hbase/client/UnmodifyableHRegionInfo.java | 2 +- .../client/UnmodifyableHTableDescriptor.java | 9 +- .../hbase/client/WrongRowIOException.java | 2 +- .../client/ZooKeeperKeepAliveConnection.java | 4 +- .../hbase/client/ZooKeeperRegistry.java | 2 + .../client/coprocessor/AggregationClient.java | 2 +- .../hbase/client/coprocessor/Batch.java | 4 +- .../BigDecimalColumnInterpreter.java | 4 +- .../coprocessor/DoubleColumnInterpreter.java | 5 +- .../coprocessor/LongColumnInterpreter.java | 2 +- .../coprocessor/SecureBulkLoadClient.java | 13 +- .../client/replication/ReplicationAdmin.java | 6 +- .../hbase/coprocessor/ColumnInterpreter.java | 4 +- .../coprocessor/CoprocessorException.java | 2 +- .../ConnectionClosingException.java | 59 -- .../exceptions/MergeRegionException.java | 2 +- .../OperationConflictException.java | 2 +- .../OutOfOrderScannerNextException.java | 2 +- .../PreemptiveFastFailException.java | 70 -- .../exceptions/RegionInRecoveryException.java | 2 +- .../exceptions/RegionMovedException.java | 4 +- .../exceptions/RegionOpeningException.java | 3 +- .../hadoop/hbase/filter/BinaryComparator.java | 3 +- .../hbase/filter/BinaryPrefixComparator.java | 3 +- .../hadoop/hbase/filter/BitComparator.java | 3 +- .../hbase/filter/ByteArrayComparable.java | 2 +- .../hbase/filter/ColumnCountGetFilter.java | 2 +- .../hbase/filter/ColumnPaginationFilter.java | 4 +- .../hbase/filter/ColumnPrefixFilter.java | 4 +- .../hbase/filter/ColumnRangeFilter.java | 4 +- .../hadoop/hbase/filter/CompareFilter.java | 5 +- .../hbase/filter/DependentColumnFilter.java | 6 +- .../hadoop/hbase/filter/FamilyFilter.java | 2 +- .../apache/hadoop/hbase/filter/Filter.java | 2 +- .../hadoop/hbase/filter/FilterBase.java | 2 +- .../hadoop/hbase/filter/FilterList.java | 4 +- .../hadoop/hbase/filter/FilterWrapper.java | 2 +- .../hbase/filter/FirstKeyOnlyFilter.java | 2 +- ...FirstKeyValueMatchingQualifiersFilter.java | 10 +- .../hadoop/hbase/filter/FuzzyRowFilter.java | 6 +- .../hbase/filter/InclusiveStopFilter.java | 4 +- .../hadoop/hbase/filter/KeyOnlyFilter.java | 4 +- .../hadoop/hbase/filter/LongComparator.java | 82 -- .../filter/MultipleColumnPrefixFilter.java | 4 +- .../hadoop/hbase/filter/PageFilter.java | 12 +- .../hadoop/hbase/filter/ParseConstants.java | 4 +- .../hadoop/hbase/filter/ParseFilter.java | 16 +- .../hadoop/hbase/filter/PrefixFilter.java | 10 +- .../hadoop/hbase/filter/QualifierFilter.java | 2 +- .../hadoop/hbase/filter/RandomRowFilter.java | 2 +- .../hbase/filter/RegexStringComparator.java | 5 +- .../apache/hadoop/hbase/filter/RowFilter.java | 2 +- .../SingleColumnValueExcludeFilter.java | 4 +- .../hbase/filter/SingleColumnValueFilter.java | 6 +- .../hadoop/hbase/filter/SkipFilter.java | 2 +- .../hbase/filter/SubstringComparator.java | 3 +- .../hadoop/hbase/filter/TimestampsFilter.java | 13 +- .../hadoop/hbase/filter/ValueFilter.java | 9 +- .../hadoop/hbase/filter/WhileMatchFilter.java | 2 +- .../hbase/ipc/CoprocessorRpcChannel.java | 8 +- .../hbase/ipc/FatalConnectionException.java | 2 +- .../ipc/MasterCoprocessorRpcChannel.java | 4 +- .../ipc/PayloadCarryingRpcController.java | 2 +- .../ipc/RegionCoprocessorRpcChannel.java | 4 +- .../RegionServerCoprocessorRpcChannel.java | 1 + .../hbase/ipc/RemoteWithExtrasException.java | 2 +- .../apache/hadoop/hbase/ipc/RpcClient.java | 119 ++- .../ipc/ServerNotRunningYetException.java | 4 +- .../hadoop/hbase/ipc/ServerRpcController.java | 7 +- .../hbase/ipc/StoppedRpcClientException.java | 2 +- .../hbase/ipc/TimeLimitedRpcController.java | 5 +- .../hadoop/hbase/master/RegionState.java | 4 +- .../hadoop/hbase/protobuf/ProtobufUtil.java | 10 +- .../hbase/protobuf/RequestConverter.java | 7 +- .../hbase/protobuf/ResponseConverter.java | 5 +- .../hadoop/hbase/quotas/QuotaRetriever.java | 6 +- .../hadoop/hbase/quotas/QuotaSettings.java | 3 +- .../hbase/quotas/QuotaSettingsFactory.java | 4 +- .../hadoop/hbase/quotas/QuotaTableUtil.java | 9 +- .../hadoop/hbase/quotas/ThrottleSettings.java | 7 +- .../hbase/quotas/ThrottlingException.java | 2 + .../hbase/regionserver/LeaseException.java | 2 +- .../NoSuchColumnFamilyException.java | 2 +- .../regionserver/WrongRegionException.java | 4 +- .../wal/FailedLogCloseException.java | 4 +- .../hbase/replication/ReplicationPeer.java | 2 +- .../replication/ReplicationPeerZKImpl.java | 4 +- .../hbase/replication/ReplicationPeers.java | 3 +- .../replication/ReplicationPeersZKImpl.java | 2 +- .../replication/ReplicationQueueInfo.java | 8 +- .../replication/ReplicationQueuesZKImpl.java | 2 +- .../hbase/replication/ReplicationTracker.java | 4 +- .../hbase/security/AccessDeniedException.java | 2 +- .../hadoop/hbase/security/AuthMethod.java | 6 +- .../hadoop/hbase/security/EncryptionUtil.java | 6 +- .../hbase/security/HBaseSaslRpcClient.java | 32 +- .../hadoop/hbase/security/SaslUtil.java | 6 +- .../hbase/security/SecureBulkLoadUtil.java | 2 +- .../security/access/AccessControlClient.java | 17 +- .../hbase/security/access/Permission.java | 13 +- .../security/access/TablePermission.java | 12 +- .../hbase/security/access/UserPermission.java | 10 +- .../token/AuthenticationTokenIdentifier.java | 9 +- .../token/AuthenticationTokenSelector.java | 4 +- .../visibility/InvalidLabelException.java | 2 +- .../LabelAlreadyExistsException.java | 2 +- .../security/visibility/VisibilityClient.java | 26 +- .../visibility/VisibilityConstants.java | 2 +- .../ClientSnapshotDescriptionUtils.java | 2 +- .../snapshot/HBaseSnapshotException.java | 2 +- .../snapshot/TablePartiallyOpenException.java | 6 +- .../apache/hadoop/hbase/util/HasThread.java | 4 +- .../org/apache/hadoop/hbase/util/Sleeper.java | 2 +- .../apache/hadoop/hbase/util/Writables.java | 8 +- .../hadoop/hbase/zookeeper/HQuorumPeer.java | 26 +- .../hbase/zookeeper/MasterAddressTracker.java | 8 +- .../hbase/zookeeper/MetaTableLocator.java | 10 +- .../hadoop/hbase/zookeeper/ZKClusterId.java | 2 +- .../hadoop/hbase/zookeeper/ZKConfig.java | 5 +- .../hbase/zookeeper/ZKLeaderManager.java | 2 +- .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 6 +- .../hbase/zookeeper/ZooKeeperNodeTracker.java | 2 +- .../hbase/zookeeper/ZooKeeperWatcher.java | 2 +- .../hadoop/hbase/TestHColumnDescriptor.java | 4 +- .../hadoop/hbase/TestHTableDescriptor.java | 35 - .../hadoop/hbase/client/TestAsyncProcess.java | 46 +- .../hbase/client/TestClientNoCluster.java | 8 +- .../client/TestFastFailWithoutTestUtil.java | 613 --------------- .../apache/hadoop/hbase/client/TestGet.java | 4 +- .../hadoop/hbase/client/TestOperation.java | 18 +- .../apache/hadoop/hbase/client/TestScan.java | 4 +- .../hbase/client/TestSnapshotFromAdmin.java | 4 +- .../hbase/filter/TestLongComparator.java | 37 - .../apache/hadoop/hbase/ipc/TestIPCUtil.java | 6 +- .../hbase/security/TestEncryptionUtil.java | 9 +- hbase-common/pom.xml | 33 +- .../java/org/apache/hadoop/hbase/Cell.java | 2 +- .../apache/hadoop/hbase/CellComparator.java | 137 +--- .../java/org/apache/hadoop/hbase/CellKey.java | 69 ++ .../org/apache/hadoop/hbase/CellScanner.java | 1 + .../org/apache/hadoop/hbase/CellUtil.java | 145 +--- .../hadoop/hbase/CompoundConfiguration.java | 13 +- .../hadoop/hbase/HBaseConfiguration.java | 19 +- .../apache/hadoop/hbase/HBaseIOException.java | 4 +- .../hadoop/hbase/HBaseInterfaceAudience.java | 8 +- .../org/apache/hadoop/hbase/HConstants.java | 43 +- .../org/apache/hadoop/hbase/KeyValue.java | 10 +- .../org/apache/hadoop/hbase/KeyValueUtil.java | 2 +- .../hadoop/hbase/NamespaceDescriptor.java | 8 +- .../hadoop/hbase/SettableSequenceId.java | 4 +- .../hadoop/hbase/SettableTimestamp.java | 6 +- .../org/apache/hadoop/hbase/TableName.java | 10 +- .../java/org/apache/hadoop/hbase/Tag.java | 21 +- .../hadoop/hbase/VersionAnnotation.java | 5 +- .../hadoop/hbase/codec/BaseDecoder.java | 6 +- .../hadoop/hbase/codec/BaseEncoder.java | 2 +- .../apache/hadoop/hbase/codec/CellCodec.java | 2 +- .../hadoop/hbase/codec/CellCodecWithTags.java | 2 +- .../org/apache/hadoop/hbase/codec/Codec.java | 2 +- .../hadoop/hbase/codec/CodecException.java | 2 +- .../hadoop/hbase/codec/KeyValueCodec.java | 2 +- .../hbase/codec/KeyValueCodecWithTags.java | 2 +- .../hbase/io/ByteBufferInputStream.java | 101 --- .../hadoop/hbase/io/CellOutputStream.java | 4 +- .../hbase/io/ImmutableBytesWritable.java | 2 +- .../hadoop/hbase/io/LimitInputStream.java | 6 +- .../hadoop/hbase/io/SizedCellScanner.java | 3 +- .../hbase/io/TagCompressionContext.java | 2 +- .../hadoop/hbase/io/compress/Compression.java | 4 +- .../hbase/io/crypto/CipherProvider.java | 2 +- .../hadoop/hbase/io/crypto/Context.java | 4 +- .../io/crypto/DefaultCipherProvider.java | 4 +- .../hadoop/hbase/io/crypto/Encryption.java | 4 +- .../hbase/io/crypto/KeyStoreKeyProvider.java | 2 +- .../io/encoding/BufferedDataBlockEncoder.java | 40 +- .../hbase/io/encoding/CompressionState.java | 2 +- .../io/encoding/CopyKeyDataBlockEncoder.java | 13 +- .../hbase/io/encoding/DataBlockEncoder.java | 2 +- .../io/encoding/DiffKeyDeltaEncoder.java | 5 +- .../hbase/io/encoding/EncodedDataBlock.java | 2 +- .../hbase/io/encoding/EncodingState.java | 2 +- .../io/encoding/FastDiffDeltaEncoder.java | 10 +- .../encoding/HFileBlockDecodingContext.java | 6 +- .../HFileBlockDefaultDecodingContext.java | 9 +- .../io/encoding/PrefixKeyDeltaEncoder.java | 8 +- .../hadoop/hbase/io/hfile/BlockType.java | 8 +- .../hadoop/hbase/io/hfile/HFileContext.java | 2 +- .../hbase/io/hfile/HFileContextBuilder.java | 2 +- .../hbase/io/util/HeapMemorySizeUtil.java | 2 +- .../apache/hadoop/hbase/security/User.java | 2 +- .../hadoop/hbase/security/UserProvider.java | 2 +- .../org/apache/hadoop/hbase/types/PBType.java | 5 +- .../org/apache/hadoop/hbase/util/Base64.java | 10 +- .../hadoop/hbase/util/ByteBufferUtils.java | 57 +- .../hadoop/hbase/util/ChecksumFactory.java | 3 +- .../hadoop/hbase/util/ConcatenatedLists.java | 1 + .../hadoop/hbase/util/ConcurrentIndex.java | 6 +- .../hbase/util/CoprocessorClassLoader.java | 2 +- .../hadoop/hbase/util/DynamicClassLoader.java | 2 +- .../org/apache/hadoop/hbase/util/Hash.java | 2 +- .../hadoop/hbase/util/PrettyPrinter.java | 2 +- .../apache/hadoop/hbase/util/VersionInfo.java | 8 +- .../hbase/util/test/RedundantKVGenerator.java | 2 +- .../apache/hadoop/hbase/ClassTestFinder.java | 2 + .../hbase/HBaseCommonTestingUtility.java | 4 +- .../apache/hadoop/hbase/ResourceChecker.java | 5 +- .../hbase/ResourceCheckerJUnitListener.java | 3 +- .../hadoop/hbase/TestCellComparator.java | 64 +- .../org/apache/hadoop/hbase/TestCellUtil.java | 29 +- .../java/org/apache/hadoop/hbase/Waiter.java | 2 +- .../hadoop/hbase/codec/TestCellCodec.java | 4 +- .../hbase/codec/TestCellCodecWithTags.java | 2 +- .../codec/TestKeyValueCodecWithTags.java | 2 +- .../hbase/io/TestByteBufferInputStream.java | 82 -- .../hbase/io/TestTagCompressionContext.java | 4 +- .../io/crypto/KeyProviderForTesting.java | 1 - .../hbase/io/crypto/TestCipherProvider.java | 7 +- .../hbase/io/crypto/TestEncryption.java | 3 +- .../hbase/io/crypto/TestKeyProvider.java | 3 +- .../hadoop/hbase/io/crypto/aes/TestAES.java | 9 +- .../hbase/io/util/TestLRUDictionary.java | 4 +- .../apache/hadoop/hbase/util/TestBase64.java | 1 - .../TestByteRangeWithKVSerialization.java | 2 +- .../hbase/util/TestConcatenatedLists.java | 7 +- .../hadoop/hbase/util/TestDrainBarrier.java | 6 +- .../util/TestEnvironmentEdgeManager.java | 17 +- .../hbase/util/TestFastLongHistogram.java | 2 +- .../hadoop/hbase/util/TestKeyLocker.java | 4 +- .../hadoop/hbase/util/TestShowProperties.java | 4 +- .../apache/hadoop/hbase/util/TestThreads.java | 4 +- hbase-examples/pom.xml | 33 +- hbase-hadoop-compat/pom.xml | 21 +- .../ipc/MetricsHBaseServerSourceFactory.java | 4 +- hbase-hadoop2-compat/pom.xml | 43 +- .../MetricsHBaseServerSourceFactoryImpl.java | 2 +- hbase-it/pom.xml | 19 - .../hadoop/hbase/IntegrationTestBase.java | 3 - .../IntegrationTestRegionReplicaPerf.java | 141 ++-- .../hadoop/hbase/chaos/actions/Action.java | 32 +- .../hbase/chaos/factories/MonkeyFactory.java | 2 - .../chaos/factories/NoKillMonkeyFactory.java | 87 --- .../SlowDeterministicMonkeyFactory.java | 1 + .../hbase/chaos/monkies/CalmChaosMonkey.java | 5 - .../hbase/chaos/monkies/ChaosMonkey.java | 6 - .../chaos/monkies/PolicyBasedChaosMonkey.java | 6 - .../policies/TwoConcurrentActionPolicy.java | 97 --- .../hbase/mttr/IntegrationTestMTTR.java | 53 +- .../test/IntegrationTestBigLinkedList.java | 33 - hbase-prefix-tree/pom.xml | 43 +- .../codec/prefixtree/PrefixTreeSeeker.java | 28 +- .../decode/PrefixTreeArrayScanner.java | 6 +- .../prefixtree/decode/PrefixTreeCell.java | 2 +- .../codec/keyvalue/TestKeyValueTool.java | 2 +- .../row/TestPrefixTreeSearcher.java | 2 +- .../codec/prefixtree/row/TestRowData.java | 2 +- .../generated/ClusterStatusProtos.java | 168 +--- .../protobuf/generated/ComparatorProtos.java | 597 +------------- .../src/main/protobuf/ClusterStatus.proto | 3 - .../src/main/protobuf/Comparator.proto | 4 - hbase-rest/pom.xml | 2 +- .../resources/hbase-webapps/rest/rest.jsp | 2 +- hbase-server/pom.xml | 28 +- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 4 +- .../tmpl/regionserver/RSStatusTmpl.jamon | 12 +- .../tmpl/regionserver/RegionListTmpl.jamon | 2 - .../hbase/CoordinatedStateManagerFactory.java | 7 +- .../hadoop/hbase/DaemonThreadFactory.java | 8 +- .../apache/hadoop/hbase/HealthCheckChore.java | 2 +- .../org/apache/hadoop/hbase/JMXListener.java | 2 +- .../apache/hadoop/hbase/TableDescriptors.java | 10 - .../apache/hadoop/hbase/TagRewriteCell.java | 202 ----- .../hbase/client/ClientSideRegionScanner.java | 2 +- .../hadoop/hbase/client/HTableWrapper.java | 2 - .../ZKSplitLogManagerCoordination.java | 4 +- .../hbase/coprocessor/BulkLoadObserver.java | 54 -- .../hbase/coprocessor/CoprocessorHost.java | 20 - .../hbase/coprocessor/RegionObserver.java | 4 - .../hadoop/hbase/coprocessor/WALObserver.java | 3 - .../org/apache/hadoop/hbase/io/FileLink.java | 29 +- .../org/apache/hadoop/hbase/io/HFileLink.java | 9 + .../hbase/io/hfile/AbstractHFileWriter.java | 16 +- .../hadoop/hbase/io/hfile/CacheConfig.java | 12 +- .../hadoop/hbase/io/hfile/ChecksumUtil.java | 7 +- .../apache/hadoop/hbase/io/hfile/HFile.java | 28 +- .../hadoop/hbase/io/hfile/HFileBlock.java | 219 +++--- .../hbase/io/hfile/HFileBlockIndex.java | 18 +- .../hbase/io/hfile/HFilePrettyPrinter.java | 12 +- .../hadoop/hbase/io/hfile/HFileReaderV2.java | 49 +- .../hadoop/hbase/io/hfile/HFileReaderV3.java | 2 +- .../hadoop/hbase/io/hfile/HFileWriterV2.java | 40 +- .../hbase/io/hfile/bucket/BucketCache.java | 2 +- .../ipc/MetricsHBaseServerWrapperImpl.java | 16 +- .../apache/hadoop/hbase/ipc/RpcServer.java | 44 +- .../hadoop/hbase/mapreduce/CellCounter.java | 9 +- .../hadoop/hbase/mapreduce/CopyTable.java | 47 +- .../mapreduce/LoadIncrementalHFiles.java | 4 + .../hbase/mapreduce/TableInputFormat.java | 31 +- .../hbase/mapreduce/TableMapReduceUtil.java | 25 +- .../hbase/mapreduce/TableOutputFormat.java | 33 +- .../{ => hadoopbackport}/JarFinder.java | 5 +- .../apache/hadoop/hbase/master/HMaster.java | 23 +- .../hbase/master/HMasterCommandLine.java | 17 +- .../hadoop/hbase/master/MasterFileSystem.java | 2 +- .../master/balancer/BaseLoadBalancer.java | 2 +- .../hbase/master/cleaner/CleanerChore.java | 2 +- .../master/handler/CreateTableHandler.java | 5 +- .../hbase/monitoring/StateDumpServlet.java | 6 +- .../protobuf/ReplicationProtbufUtil.java | 2 +- .../hadoop/hbase/quotas/QuotaCache.java | 2 - .../DelimitedKeyPrefixRegionSplitPolicy.java | 22 +- .../hadoop/hbase/regionserver/HRegion.java | 2 +- .../hbase/regionserver/HRegionServer.java | 7 +- .../hadoop/hbase/regionserver/HStore.java | 9 +- .../KeyPrefixRegionSplitPolicy.java | 50 +- .../hbase/regionserver/RSRpcServices.java | 4 +- .../regionserver/RegionServerServices.java | 6 +- .../hadoop/hbase/regionserver/ScanInfo.java | 7 +- .../hbase/regionserver/ScanQueryMatcher.java | 19 +- .../hbase/regionserver/StoreScanner.java | 4 +- .../hadoop/hbase/regionserver/wal/FSHLog.java | 2 +- .../hbase/regionserver/wal/HLogSplitter.java | 11 +- .../regionserver/ReplicationSinkManager.java | 4 +- .../regionserver/ReplicationSource.java | 7 +- .../ReplicationSourceManager.java | 4 +- .../security/access/AccessControlLists.java | 39 +- .../security/access/AccessController.java | 55 +- .../access/SecureBulkLoadEndpoint.java | 36 +- .../security/access/ZKPermissionWatcher.java | 31 +- .../visibility/VisibilityController.java | 26 +- .../hadoop/hbase/util/AbstractHBaseTool.java | 2 +- .../hadoop/hbase/util/ByteBloomFilter.java | 14 +- .../hbase/util/CompoundBloomFilter.java | 2 +- .../hadoop/hbase/util/CompressionTest.java | 2 +- .../hadoop/hbase/util/FSTableDescriptors.java | 244 +++--- .../org/apache/hadoop/hbase/util/FSUtils.java | 4 +- .../apache/hadoop/hbase/util/HBaseFsck.java | 23 +- .../hadoop/hbase/util/MultiHConnection.java | 20 +- .../hbase/util/YammerHistogramUtils.java | 80 -- .../hbase/zookeeper/MiniZooKeeperCluster.java | 8 +- .../resources/hbase-webapps/master/table.jsp | 11 +- .../org/apache/hadoop/hbase/HBaseCluster.java | 5 +- .../apache/hadoop/hbase/HBaseTestCase.java | 7 +- .../hadoop/hbase/HBaseTestingUtility.java | 2 +- .../hadoop/hbase/PerformanceEvaluation.java | 114 +-- .../apache/hadoop/hbase/TestIOFencing.java | 15 +- .../client/HConnectionTestingUtility.java | 3 - .../{TestAdmin1.java => TestAdmin.java} | 651 +++++++++++++++- .../hadoop/hbase/client/TestAdmin2.java | 734 ------------------ .../hadoop/hbase/client/TestFastFail.java | 313 -------- .../hbase/client/TestFromClientSide.java | 40 +- .../hbase/client/TestFromClientSide3.java | 4 +- .../hbase/client/TestMultiParallel.java | 13 +- .../coprocessor/SampleRegionWALObserver.java | 4 +- .../hbase/coprocessor/TestClassLoading.java | 13 - ...stMasterCoprocessorExceptionWithAbort.java | 2 +- .../hbase/coprocessor/TestWALObserver.java | 30 - .../hadoop/hbase/filter/TestFilter.java | 1 + .../filter/TestSingleColumnValueFilter.java | 20 - .../apache/hadoop/hbase/io/TestHeapSize.java | 5 +- .../hadoop/hbase/io/hfile/TestChecksum.java | 14 +- .../hadoop/hbase/io/hfile/TestHFile.java | 14 +- .../hadoop/hbase/io/hfile/TestHFileBlock.java | 10 +- .../io/hfile/TestHFileBlockCompatibility.java | 10 +- .../hbase/io/hfile/TestHFileBlockIndex.java | 2 +- .../hbase/io/hfile/TestHFileEncryption.java | 4 +- .../hbase/io/hfile/TestHFileWriterV2.java | 2 +- .../hbase/io/hfile/TestHFileWriterV3.java | 2 +- .../hfile/TestScannerSelectionUsingTTL.java | 3 +- .../hadoop/hbase/io/hfile/TestSeekTo.java | 21 +- .../hadoop/hbase/ipc/TestRpcMetrics.java | 4 +- .../hadoop/hbase/mapred/TestDriver.java | 1 + .../hbase/mapreduce/TestCellCounter.java | 54 -- .../mapreduce/TestHFileOutputFormat.java | 2 +- .../mapreduce/TestLoadIncrementalHFiles.java | 28 - ...estLoadIncrementalHFilesSplitRecovery.java | 2 +- .../TestTableSnapshotInputFormat.java | 21 +- .../{ => hadoopbackport}/TestJarFinder.java | 5 +- .../hbase/master/TestCatalogJanitor.java | 8 - .../master/TestDistributedLogSplitting.java | 4 +- .../hbase/master/TestSplitLogManager.java | 2 +- .../hbase/regionserver/TestBlocksRead.java | 6 +- .../hbase/regionserver/TestBlocksScanned.java | 3 +- .../regionserver/TestDefaultMemStore.java | 8 +- .../TestEndToEndSplitTransaction.java | 2 +- .../TestGetClosestAtOrBefore.java | 4 - .../hbase/regionserver/TestKeepDeletes.java | 147 +--- .../hbase/regionserver/TestKeyValueHeap.java | 6 - .../hbase/regionserver/TestMinVersions.java | 19 +- .../hbase/regionserver/TestQueryMatcher.java | 24 +- .../regionserver/TestRegionReplicas.java | 69 +- .../TestRegionServerNoMaster.java | 34 + .../regionserver/TestReversibleScanners.java | 3 +- .../hbase/regionserver/TestRowTooBig.java | 12 +- .../regionserver/TestScanDeleteTracker.java | 11 - .../TestScanWildcardColumnTracker.java | 3 - .../hadoop/hbase/regionserver/TestStore.java | 6 +- .../hbase/regionserver/TestStoreFile.java | 41 +- .../hbase/regionserver/TestStoreFileInfo.java | 2 - .../hbase/regionserver/TestStoreScanner.java | 9 +- .../hadoop/hbase/regionserver/TestTags.java | 94 ++- .../hbase/regionserver/TestWideScanner.java | 2 - .../hbase/regionserver/wal/TestHLog.java | 14 +- .../hbase/regionserver/wal/TestHLogSplit.java | 8 +- .../replication/TestReplicationBase.java | 4 - .../TestReplicationSourceManager.java | 2 - .../security/access/TestAccessController.java | 6 +- .../access/TestScanEarlyTermination.java | 3 +- .../hbase/snapshot/SnapshotTestingUtils.java | 1 + .../snapshot/TestFlushSnapshotFromClient.java | 8 +- .../hbase/util/TestFSTableDescriptors.java | 121 --- .../hadoop/hbase/util/TestHBaseFsck.java | 18 - .../hadoop/hbase/util/TestMergeTool.java | 13 +- hbase-shell/pom.xml | 33 +- hbase-shell/src/main/ruby/hbase/admin.rb | 97 +-- hbase-shell/src/main/ruby/hbase/security.rb | 4 +- hbase-shell/src/main/ruby/shell.rb | 9 - .../src/main/ruby/shell/commands/describe.rb | 11 +- .../ruby/shell/commands/update_all_config.rb | 39 - .../main/ruby/shell/commands/update_config.rb | 40 - .../ruby/shell/commands/user_permission.rb | 2 +- hbase-thrift/pom.xml | 43 +- .../hadoop/hbase/thrift2/HTablePool.java | 3 - pom.xml | 119 +-- src/main/docbkx/configuration.xml | 44 +- src/main/docbkx/developer.xml | 69 +- src/main/docbkx/ops_mgt.xml | 159 +--- src/main/docbkx/troubleshooting.xml | 14 - src/main/docbkx/upgrading.xml | 95 +-- 528 files changed, 2971 insertions(+), 9231 deletions(-) delete mode 100755 dev-support/make_patch.sh delete mode 100755 dev-support/rebase_all_git_branches.sh delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/FailureInfo.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryableCallerInterceptor.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptor.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorFactory.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ConnectionClosingException.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/PreemptiveFastFailException.java delete mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/filter/LongComparator.java delete mode 100644 hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestFastFailWithoutTestUtil.java delete mode 100644 hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/CellKey.java delete mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferInputStream.java delete mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBufferInputStream.java delete mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java delete mode 100644 hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/TwoConcurrentActionPolicy.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/TagRewriteCell.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java rename hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/{ => hadoopbackport}/JarFinder.java (98%) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java rename hbase-server/src/test/java/org/apache/hadoop/hbase/client/{TestAdmin1.java => TestAdmin.java} (62%) delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java rename hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/{ => hadoopbackport}/TestJarFinder.java (96%) delete mode 100644 hbase-shell/src/main/ruby/shell/commands/update_all_config.rb delete mode 100644 hbase-shell/src/main/ruby/shell/commands/update_config.rb diff --git a/bin/hbase.cmd b/bin/hbase.cmd index 1b2227b9644f..2e75c38bb23b 100644 --- a/bin/hbase.cmd +++ b/bin/hbase.cmd @@ -239,9 +239,7 @@ if "%servercommand%" == "true" ( if defined service_entry ( set HBASE_LOG_PREFIX=hbase-%hbase-command%-%COMPUTERNAME% set HBASE_LOGFILE=!HBASE_LOG_PREFIX!.log - if not defined HBASE_ROOT_LOGGER ( - set HBASE_ROOT_LOGGER=INFO,DRFA - ) + set HBASE_ROOT_LOGGER=INFO,DRFA set HBASE_SECURITY_LOGGER=INFO,DRFAS set loggc=!HBASE_LOG_DIR!\!HBASE_LOG_PREFIX!.gc set loglog=!HBASE_LOG_DIR!\!HBASE_LOGFILE! diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh deleted file mode 100755 index ec2bbb9dd380..000000000000 --- a/dev-support/make_patch.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Make a patch for the current branch based on its tracking branch - -# Process args -while getopts "ahd:" opt; do - case "$opt" in - a) addendum='-addendum' - ;; - d) - patch_dir=$OPTARG - ;; - *) - echo -e "Usage: $0 [-h] [-a] [-d] \n\ - Must be run from within the git branch to make the patch against.\n\ - -h - display these instructions.\n\ - -a - Add an 'addendum' prefix to the patch name.\n\ - -d - specify a patch directory (defaults to ~/patches/)" - exit 0 - ;; - esac -done - -# Find what branch we are on -branch=$(git branch |grep '*' |awk '{print $2}') -if [ ! "$branch" ]; then - echo "Can't determine the git branch. Exiting." >&2 - exit 1 -fi - -# Exit if git status is dirty -git_dirty=$(git diff --shortstat 2> /dev/null | wc -l|awk {'print $1'}) -echo "git_dirty is $git_dirty" -if [ "$git_dirty" -ne 0 ]; then - echo "Git status is dirty. Commit locally first.">&2 - exit 1 -fi - -# Determine the tracking branch -git log -n 1 origin/$branch > /dev/null 2>&1 -status=$? -if [ "$status" -eq 128 ]; then - # Status 128 means there is no remote branch - tracking_branch='origin/master' -elif [ "$status" -eq 0 ]; then - # Status 0 means there is a remote branch - tracking_branch="origin/$branch" -else - echo "Unknown error: $?" >&2 - exit 1 -fi - -# Deal with invalid or missing $patch_dir -if [ ! "$patch_dir" ]; then - echo -e "Patch directory not specified. Falling back to ~/patches/." - patch_dir=~/patches -fi - -if [ ! -d "$patch_dir" ]; then - echo "$patch_dir does not exist. Creating it." - mkdir $patch_dir -fi - -# Determine what to call the patch -# Check to see if any patch exists that includes the branch name -status=$(ls $patch_dir/*$branch* 2>/dev/null|grep -v addendum|wc -l|awk {'print $1'}) -if [ "$status" -eq 0 ]; then - # This is the first patch we are making for this release - prefix='' -elif [ "$status" -ge 1 ]; then - # At least one patch already exists -- add a version prefix - for i in {1..99}; do - # Check to see the maximum version of patch that exists - if [ ! -f "$patch_dir/$branch-v$i.patch" ]; then - version=$i - if [ -n "$addendum" ]; then - # Don't increment the patch # if it is an addendum - echo "Creating an addendum" - if [ "$version" -eq 1 ]; then - # We are creating an addendum to the first version of the patch - prefix='' - else - # We are making an addendum to a different version of the patch - let version=$version-1 - prefix="-v$version" - fi - else - prefix="-v$version" - fi - break - fi - done -fi - -patch_name="$branch$prefix$addendum.patch" - -# Do we need to make a diff? -git diff --quiet $tracking_branch -status=$? -if [ "$status" -eq 0 ]; then - echo "There is no difference between $branch and $tracking_branch." - echo "No patch created." - exit 0 -fi - -# Check whether we need to squash or not -local_commits=$(git log $tracking_branch..$branch|grep 'Author:'|wc -l|awk {'print $1'}) -if [ "$local_commits" -gt 1 ]; then - read -p "$local_commits commits exist only in your local branch. Interactive rebase?" yn - case $yn in - [Yy]* ) - git rebase -i $tracking_branch - ;; - [Nn]* ) - echo "Creating $patch_dir/$patch_name using git diff." - git diff $tracking_branch > $patch_dir/$patch_name - exit 0 - ;; - esac -fi - -echo "Creating patch $patch_dir/$patch_name using git format-patch" -git format-patch --stdout $tracking_branch > $patch_dir/$patch_name - - diff --git a/dev-support/rebase_all_git_branches.sh b/dev-support/rebase_all_git_branches.sh deleted file mode 100755 index 19364709e601..000000000000 --- a/dev-support/rebase_all_git_branches.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script assumes that your remote is called "origin" -# and that your local master branch is called "master". -# I am sure it could be made more abstract but these are the defaults. - -# Edit this line to point to your default directory, -# or always pass a directory to the script. - -DEFAULT_DIR="EDIT_ME" - -function print_usage { - cat << __EOF - -$0: A script to manage your Apache HBase Git repository. - -If run with no arguments, it reads the DEFAULT_DIR variable, which you -can specify by editing the script. - -Usage: $0 [-d ] - $0 -h - - -h Show this screen. - -d The absolute or relative directory of your Git repository. -__EOF -} - -function get_all_branches { - # Gets all git branches present locally - all_branches=() - for i in `git branch --list | sed -e "s/\*//g"`; do - all_branches+=("$(echo $i | awk '{print($1)}')") - done -} - -function get_tracking_branches { - # Gets all branches with a remote tracking branch - tracking_branches=() - for i in `git branch -lvv | grep "\[origin/" | sed -e 's/\*//g' | awk {'print $1'}`; do - tracking_branches+=("$(echo $i | awk '{print($1)}')") - done -} - -function check_git_branch_status { - # Checks the current Git branch to see if it's dirty - # Returns 1 if the branch is dirty - git_dirty=$(git diff --shortstat 2> /dev/null | wc -l|awk {'print $1'}) - if [ "$git_dirty" -ne 0 ]; then - echo "Git status is dirty. Commit locally first." >&2 - exit 1 - fi -} - -function get_jira_status { - # This function expects as an argument the JIRA ID, - # and returns 99 if resolved and 1 if it couldn't - # get the status. - - # The JIRA status looks like this in the HTML: - # span id="resolution-val" class="value resolved" > - # The following is a bit brittle, but filters for lines with - # resolution-val returns 99 if it's resolved - jira_url='https://issues.apache.org/jira/browse' - jira_id="$1" - status="$(curl -s $jira_url/$jira_id | \ - grep resolution-val | \ - sed -e "s/.*class=\"value\ //" | \ - cut -d'"' -f 1)" - if [ $? -ne 0 ]; then - echo "Could not get JIRA status. Check your network." >&2 - exit 1 - fi - if [ "$status" = "resolved" ]; then - return 99 - fi -} - -# Process the arguments -while getopts ":hd:" opt; do - case $opt in - d) - # A directory was passed in - dir="$OPTARG" - if [ ! -d "$dir/.git/" ]; then - echo "$dir does not exist or is not a Git repository." >&2 - exit 1 - fi - ;; - h) - # Print usage instructions - print_usage - exit 0 - ;; - *) - echo "Invalid argument: $OPTARG" >&2 - print_usage >&2 - exit 1 - ;; - esac -done - -if [ -z "$dir" ]; then - # No directory was passed in - dir="$DEFAULT_DIR" - if [ "$dir" = "EDIT_ME" ]; then - echo "You need to edit the DEFAULT_DIR in $0." >&2 - $0 -h - exit 1 - elif [ ! -d "$DEFAULT_DIR/.git/" ]; then - echo "Default directory $DEFAULT_DIR is not a Git repository." >&2 - exit 1 - fi -fi - -cd "$dir" - -# For each tracking branch, check it out and make sure it's fresh -# This function creates tracking_branches array and stores the tracking branches in it -get_tracking_branches -for i in "${tracking_branches[@]}"; do - git checkout -q "$i" - # Exit if git status is dirty - check_git_branch_status - git pull -q --rebase - status=$? - if [ "$status" -ne 0 ]; then - echo "Unable to pull changes in $i: $status Exiting." >&2 - exit 1 - fi - echo "Refreshed $i from remote." -done - -# Run the function to get the list of all branches -# The function creates array all_branches and stores the branches in it -get_all_branches - -# Declare array to hold deleted branch info -deleted_branches=() - -for i in "${all_branches[@]}"; do - # Check JIRA status to see if we still need this branch - # JIRA expects uppercase - jira_id="$(echo $i | awk '{print toupper($0)'})" - if [[ "$jira_id" == HBASE-* ]]; then - # Returns 1 if the JIRA is closed, 0 otherwise - get_jira_status "$jira_id" - jira_status=$? - if [ $jira_status -eq 99 ]; then - # the JIRA seems to be resolved or is at least not unresolved - deleted_branches+=("$i") - fi - fi - - git checkout -q "$i" - - # Exit if git status is dirty - check_git_branch_status - - # If this branch has a remote, don't rebase it - # If it has a remote, it has a log with at least one entry - git log -n 1 origin/"$i" > /dev/null 2>&1 - status=$? - if [ $status -eq 128 ]; then - # Status 128 means there is no remote branch - # Try to rebase against master - echo "Rebasing $i on origin/master" - git rebase -q origin/master > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "Failed. Rolling back. Rebase $i manually." - git rebase --abort - fi - elif [ $status -ne 0 ]; then - # If status is 0 it means there is a remote branch, we already took care of it - echo "Unknown error: $?" >&2 - exit 1 - fi -done - -# Offer to clean up all deleted branches -for i in "${deleted_branches[@]}"; do - read -p "$i's JIRA is resolved. Delete? " yn - case $yn in - [Yy]) - git branch -D $i - ;; - *) - echo "To delete it manually, run git branch -D $deleted_branches" - ;; - esac -done -git checkout -q master -exit 0 diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index bf68f3e0772f..73816562474f 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -517,18 +517,14 @@ checkCheckstyleErrors() { JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 checkstyle{color}. The applied patch generated $patchCheckstyleErrors checkstyle errors (more than the trunk's current $trunkCheckstyleErrors errors)." + {color:red}-1 javac{color}. The applied patch generated $patchCheckstyleErrors checkstyle errors (more than the trunk's current $trunkCheckstyleErrors errors)." return 1 fi echo "There were $patchCheckstyleErrors checkstyle errors in this patch compared to $trunkCheckstyleErrors on master." fi - JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html - - $JIRA_COMMENT_FOOTER" - JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 checkstyle{color}. The applied patch does not increase the total number of checkstyle errors" + {color:green}+1 javac{color}. The applied patch does not increase the total number of checkstyle errors" return 0 } @@ -583,7 +579,7 @@ checkReleaseAuditWarnings () { {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)." $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditWarnings.txt + JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt $JIRA_COMMENT_FOOTER" return 1 fi diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 520e85a65fb2..f86ecab00e95 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -118,14 +118,6 @@ 0644 - - ${project.basedir}/../hbase-annotations/target/ - lib - - ${annotations.test.jar} - - 0644 - ${project.basedir}/../hbase-common/target/ lib diff --git a/hbase-checkstyle/pom.xml b/hbase-checkstyle/pom.xml index 6f3d71cfff1c..295ad9a94191 100644 --- a/hbase-checkstyle/pom.xml +++ b/hbase-checkstyle/pom.xml @@ -28,46 +28,5 @@ HBase - Checkstyle Module to hold Checkstyle properties for HBase. - - hbase - org.apache.hbase - 2.0.0-SNAPSHOT - .. - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - maven-assembly-plugin - - true - - - - org.apache.maven.plugins - maven-surefire-plugin - - - secondPartTestsExecution - test - - test - - - true - - - - - - - + \ No newline at end of file diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml index 5f3f58687744..3531e2a5701c 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle-suppressions.xml @@ -2,35 +2,7 @@ - - + \ No newline at end of file diff --git a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml index 34fe5ec18bae..8f58623aa9f4 100644 --- a/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml +++ b/hbase-checkstyle/src/main/resources/hbase/checkstyle.xml @@ -1,34 +1,6 @@ - @@ -57,4 +29,4 @@ - + \ No newline at end of file diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml index 5d21ea3204c0..d361e5585622 100644 --- a/hbase-client/pom.xml +++ b/hbase-client/pom.xml @@ -34,25 +34,6 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - org.apache.maven.plugins maven-site-plugin diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/Chore.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/Chore.java index 42d9d373e268..7d1346f9c236 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/Chore.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/Chore.java @@ -38,7 +38,7 @@ public abstract class Chore extends HasThread { private final Log LOG = LogFactory.getLog(this.getClass()); private final Sleeper sleeper; - private final Stoppable stopper; + protected final Stoppable stopper; /** * @param p Period at which we should run. Will be adjusted appropriately @@ -146,12 +146,4 @@ protected void sleep() { */ protected void cleanup() { } - - protected Stoppable getStopper() { - return stopper; - } - - protected Sleeper getSleeper() { - return sleeper; - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java index f835857dfb65..5be224c7206f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hbase; -import java.util.UUID; - +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClusterIdProtos; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.InvalidProtocolBufferException; +import java.util.UUID; /** * The identifier for this cluster. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java index 50eda7a8a7b2..7599e3eaf544 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.master.RegionState; @@ -37,7 +38,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.VersionedWritable; @@ -216,7 +216,6 @@ public byte getVersion() { * @return region server information * @deprecated Use {@link #getServers()} */ - @Deprecated public Collection getServerInfo() { return getServers(); } @@ -379,37 +378,47 @@ public ClusterStatusProtos.ClusterStatus convert() { public static ClusterStatus convert(ClusterStatusProtos.ClusterStatus proto) { Map servers = null; - servers = new HashMap(proto.getLiveServersList().size()); - for (LiveServerInfo lsi : proto.getLiveServersList()) { - servers.put(ProtobufUtil.toServerName( - lsi.getServer()), new ServerLoad(lsi.getServerLoad())); + if (proto.getLiveServersList() != null) { + servers = new HashMap(proto.getLiveServersList().size()); + for (LiveServerInfo lsi : proto.getLiveServersList()) { + servers.put(ProtobufUtil.toServerName( + lsi.getServer()), new ServerLoad(lsi.getServerLoad())); + } } Collection deadServers = null; - deadServers = new ArrayList(proto.getDeadServersList().size()); - for (HBaseProtos.ServerName sn : proto.getDeadServersList()) { - deadServers.add(ProtobufUtil.toServerName(sn)); + if (proto.getDeadServersList() != null) { + deadServers = new ArrayList(proto.getDeadServersList().size()); + for (HBaseProtos.ServerName sn : proto.getDeadServersList()) { + deadServers.add(ProtobufUtil.toServerName(sn)); + } } Collection backupMasters = null; - backupMasters = new ArrayList(proto.getBackupMastersList().size()); - for (HBaseProtos.ServerName sn : proto.getBackupMastersList()) { - backupMasters.add(ProtobufUtil.toServerName(sn)); + if (proto.getBackupMastersList() != null) { + backupMasters = new ArrayList(proto.getBackupMastersList().size()); + for (HBaseProtos.ServerName sn : proto.getBackupMastersList()) { + backupMasters.add(ProtobufUtil.toServerName(sn)); + } } Map rit = null; - rit = new HashMap(proto.getRegionsInTransitionList().size()); - for (RegionInTransition region : proto.getRegionsInTransitionList()) { - String key = new String(region.getSpec().getValue().toByteArray()); - RegionState value = RegionState.convert(region.getRegionState()); - rit.put(key, value); + if (proto.getRegionsInTransitionList() != null) { + rit = new HashMap(proto.getRegionsInTransitionList().size()); + for (RegionInTransition region : proto.getRegionsInTransitionList()) { + String key = new String(region.getSpec().getValue().toByteArray()); + RegionState value = RegionState.convert(region.getRegionState()); + rit.put(key, value); + } } String[] masterCoprocessors = null; - final int numMasterCoprocessors = proto.getMasterCoprocessorsCount(); - masterCoprocessors = new String[numMasterCoprocessors]; - for (int i = 0; i < numMasterCoprocessors; i++) { - masterCoprocessors[i] = proto.getMasterCoprocessors(i).getName(); + if (proto.getMasterCoprocessorsList() != null) { + final int numMasterCoprocessors = proto.getMasterCoprocessorsCount(); + masterCoprocessors = new String[numMasterCoprocessors]; + for (int i = 0; i < numMasterCoprocessors; i++) { + masterCoprocessors[i] = proto.getMasterCoprocessors(i).getName(); + } } return new ClusterStatus(proto.getHbaseVersion().getVersion(), diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java index 37f1a33ed719..d9e2bdce5e17 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CoprocessorEnvironment.java @@ -18,8 +18,8 @@ import java.io.IOException; import java.util.concurrent.ExecutorService; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java index 8be2518b75e7..b566fcf8b645 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/DoNotRetryIOException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; /** * Subclass if exception is not meant to be retried: e.g. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java index 6ff1b1f7ca11..c4799b2bd6c3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/DroppedSnapshotException.java @@ -14,11 +14,11 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown during flush if the possibility snapshot content was not properly diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 2f6d11e8cc61..4958fb7e3e57 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.Set; +import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; @@ -39,9 +41,6 @@ import org.apache.hadoop.hbase.util.PrettyPrinter; import org.apache.hadoop.hbase.util.PrettyPrinter.Unit; -import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; - /** * An HColumnDescriptor contains information about a column family such as the * number of versions, compression settings, etc. @@ -159,7 +158,7 @@ public class HColumnDescriptor implements Comparable { /** * Default setting for preventing deleted from being collected immediately. */ - public static final KeepDeletedCells DEFAULT_KEEP_DELETED = KeepDeletedCells.FALSE; + public static final boolean DEFAULT_KEEP_DELETED = false; /** * Default setting for whether to use a block cache or not. @@ -426,7 +425,7 @@ public HColumnDescriptor(final byte [] familyName, final int maxVersions, */ @Deprecated public HColumnDescriptor(final byte[] familyName, final int minVersions, - final int maxVersions, final KeepDeletedCells keepDeletedCells, + final int maxVersions, final boolean keepDeletedCells, final String compression, final boolean encodeOnDisk, final String dataBlockEncoding, final boolean inMemory, final boolean blockCacheEnabled, final int blocksize, @@ -632,7 +631,6 @@ public synchronized int getBlocksize() { Integer.decode(value): Integer.valueOf(DEFAULT_BLOCKSIZE); } return this.blocksize.intValue(); - } /** @@ -665,10 +663,7 @@ public HColumnDescriptor setCompressionType(Compression.Algorithm type) { return setValue(COMPRESSION, type.getName().toUpperCase()); } - /** - * @return data block encoding algorithm used on disk - * @deprecated See getDataBlockEncoding() - */ + /** @return data block encoding algorithm used on disk */ @Deprecated public DataBlockEncoding getDataBlockEncodingOnDisk() { return getDataBlockEncoding(); @@ -678,7 +673,6 @@ public DataBlockEncoding getDataBlockEncodingOnDisk() { * This method does nothing now. Flag ENCODE_ON_DISK is not used * any more. Data blocks have the same encoding in cache as on disk. * @return this (for chained invocation) - * @deprecated This does nothing now. */ @Deprecated public HColumnDescriptor setEncodeOnDisk(boolean encodeOnDisk) { @@ -726,9 +720,7 @@ public HColumnDescriptor setCompressTags(boolean compressTags) { /** * @return Whether KV tags should be compressed along with DataBlockEncoding. When no * DataBlockEncoding is been used, this is having no effect. - * @deprecated Use {@link #isCompressTags()} instead */ - @Deprecated public boolean shouldCompressTags() { String compressTagsStr = getValue(COMPRESS_TAGS); boolean compressTags = DEFAULT_COMPRESS_TAGS; @@ -738,19 +730,6 @@ public boolean shouldCompressTags() { return compressTags; } - /** - * @return Whether KV tags should be compressed along with DataBlockEncoding. When no - * DataBlockEncoding is been used, this is having no effect. - */ - public boolean isCompressTags() { - String compressTagsStr = getValue(COMPRESS_TAGS); - boolean compressTags = DEFAULT_COMPRESS_TAGS; - if (compressTagsStr != null) { - compressTags = Boolean.valueOf(compressTagsStr); - } - return compressTags; - } - /** * @return Compression type setting. */ @@ -791,11 +770,10 @@ public HColumnDescriptor setInMemory(boolean inMemory) { return setValue(HConstants.IN_MEMORY, Boolean.toString(inMemory)); } - public KeepDeletedCells getKeepDeletedCells() { + public boolean getKeepDeletedCells() { String value = getValue(KEEP_DELETED_CELLS); if (value != null) { - // toUpperCase for backwards compatibility - return KeepDeletedCells.valueOf(value.toUpperCase()); + return Boolean.valueOf(value).booleanValue(); } return DEFAULT_KEEP_DELETED; } @@ -804,21 +782,9 @@ public KeepDeletedCells getKeepDeletedCells() { * @param keepDeletedCells True if deleted rows should not be collected * immediately. * @return this (for chained invocation) - * @deprecated use {@link #setKeepDeletedCells(KeepDeletedCells)} */ - @Deprecated public HColumnDescriptor setKeepDeletedCells(boolean keepDeletedCells) { - return setValue(KEEP_DELETED_CELLS, (keepDeletedCells ? KeepDeletedCells.TRUE - : KeepDeletedCells.FALSE).toString()); - } - - /** - * @param keepDeletedCells True if deleted rows should not be collected - * immediately. - * @return this (for chained invocation) - */ - public HColumnDescriptor setKeepDeletedCells(KeepDeletedCells keepDeletedCells) { - return setValue(KEEP_DELETED_CELLS, keepDeletedCells.toString()); + return setValue(KEEP_DELETED_CELLS, Boolean.toString(keepDeletedCells)); } /** @@ -914,20 +880,11 @@ public HColumnDescriptor setScope(int scope) { /** * @return true if we should cache data blocks on write - * @deprecated Use {@link #isCacheDataOnWrite()} instead */ - @Deprecated public boolean shouldCacheDataOnWrite() { return setAndGetBoolean(CACHE_DATA_ON_WRITE, DEFAULT_CACHE_DATA_ON_WRITE); } - /** - * @return true if we should cache data blocks on write - */ - public boolean isCacheDataOnWrite() { - return setAndGetBoolean(CACHE_DATA_ON_WRITE, DEFAULT_CACHE_DATA_ON_WRITE); - } - /** * @param value true if we should cache data blocks on write * @return this (for chained invocation) @@ -939,21 +896,11 @@ public HColumnDescriptor setCacheDataOnWrite(boolean value) { /** * @return true if we should cache data blocks in the L1 cache (if block cache deploy * has more than one tier; e.g. we are using CombinedBlockCache). - * @deprecated Use {@link #isCacheDataInL1()} instead */ - @Deprecated public boolean shouldCacheDataInL1() { return setAndGetBoolean(CACHE_DATA_IN_L1, DEFAULT_CACHE_DATA_IN_L1); } - /** - * @return true if we should cache data blocks in the L1 cache (if block cache deploy has more - * than one tier; e.g. we are using CombinedBlockCache). - */ - public boolean isCacheDataInL1() { - return setAndGetBoolean(CACHE_DATA_IN_L1, DEFAULT_CACHE_DATA_IN_L1); - } - /** * @param value true if we should cache data blocks in the L1 cache (if block cache deploy * has more than one tier; e.g. we are using CombinedBlockCache). @@ -971,20 +918,11 @@ private boolean setAndGetBoolean(final String key, final boolean defaultSetting) /** * @return true if we should cache index blocks on write - * @deprecated Use {@link #isCacheIndexesOnWrite()} instead */ - @Deprecated public boolean shouldCacheIndexesOnWrite() { return setAndGetBoolean(CACHE_INDEX_ON_WRITE, DEFAULT_CACHE_INDEX_ON_WRITE); } - /** - * @return true if we should cache index blocks on write - */ - public boolean isCacheIndexesOnWrite() { - return setAndGetBoolean(CACHE_INDEX_ON_WRITE, DEFAULT_CACHE_INDEX_ON_WRITE); - } - /** * @param value true if we should cache index blocks on write * @return this (for chained invocation) @@ -995,20 +933,11 @@ public HColumnDescriptor setCacheIndexesOnWrite(boolean value) { /** * @return true if we should cache bloomfilter blocks on write - * @deprecated Use {@link #isCacheBloomsOnWrite()} instead */ - @Deprecated public boolean shouldCacheBloomsOnWrite() { return setAndGetBoolean(CACHE_BLOOMS_ON_WRITE, DEFAULT_CACHE_BLOOMS_ON_WRITE); } - /** - * @return true if we should cache bloomfilter blocks on write - */ - public boolean isCacheBloomsOnWrite() { - return setAndGetBoolean(CACHE_BLOOMS_ON_WRITE, DEFAULT_CACHE_BLOOMS_ON_WRITE); - } - /** * @param value true if we should cache bloomfilter blocks on write * @return this (for chained invocation) @@ -1020,20 +949,11 @@ public HColumnDescriptor setCacheBloomsOnWrite(boolean value) { /** * @return true if we should evict cached blocks from the blockcache on * close - * @deprecated {@link #isEvictBlocksOnClose()} instead */ - @Deprecated public boolean shouldEvictBlocksOnClose() { return setAndGetBoolean(EVICT_BLOCKS_ON_CLOSE, DEFAULT_EVICT_BLOCKS_ON_CLOSE); } - /** - * @return true if we should evict cached blocks from the blockcache on close - */ - public boolean isEvictBlocksOnClose() { - return setAndGetBoolean(EVICT_BLOCKS_ON_CLOSE, DEFAULT_EVICT_BLOCKS_ON_CLOSE); - } - /** * @param value true if we should evict cached blocks from the blockcache on * close @@ -1045,20 +965,11 @@ public HColumnDescriptor setEvictBlocksOnClose(boolean value) { /** * @return true if we should prefetch blocks into the blockcache on open - * @deprecated Use {@link #isPrefetchBlocksOnOpen()} instead */ - @Deprecated public boolean shouldPrefetchBlocksOnOpen() { return setAndGetBoolean(PREFETCH_BLOCKS_ON_OPEN, DEFAULT_PREFETCH_BLOCKS_ON_OPEN); } - /** - * @return true if we should prefetch blocks into the blockcache on open - */ - public boolean isPrefetchBlocksOnOpen() { - return setAndGetBoolean(PREFETCH_BLOCKS_ON_OPEN, DEFAULT_PREFETCH_BLOCKS_ON_OPEN); - } - /** * @param value true if we should prefetch blocks into the blockcache on open * @return this (for chained invocation) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index 82beb0b2d1e5..49f31d69ea90 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -18,25 +18,30 @@ */ package org.apache.hadoop.hbase; +import java.io.ByteArrayInputStream; +import java.io.DataInput; import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.EOFException; import java.io.IOException; +import java.io.SequenceInputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JenkinsHash; import org.apache.hadoop.hbase.util.MD5Hash; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java index edb53dc0bb8e..373e76b57c1e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java @@ -104,8 +104,7 @@ public long getSeqNum() { } /** - * @return String made of hostname and port formatted as - * per {@link Addressing#createHostAndPortStr(String, int)} + * @return String made of hostname and port formatted as per {@link Addressing#createHostAndPortStr(String, int)} */ public String getHostnamePort() { return Addressing.createHostAndPortStr(this.getHostname(), this.getPort()); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index bfd33009d618..058819bdf9b9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -18,6 +18,7 @@ */ package org.apache.hadoop.hbase; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -32,13 +33,12 @@ import java.util.TreeSet; import java.util.regex.Matcher; -import javax.annotation.Nonnull; - +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -51,8 +51,6 @@ import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.InvalidProtocolBufferException; - /** * HTableDescriptor contains the details about an HBase table such as the descriptors of * all the column families, is the table a catalog table, -ROOT- or @@ -767,33 +765,11 @@ public HTableDescriptor setMemStoreFlushSize(long memstoreFlushSize) { /** * Adds a column family. - * For the updating purpose please use {@link #modifyFamily(HColumnDescriptor)} instead. * @param family HColumnDescriptor of family to add. */ public HTableDescriptor addFamily(final HColumnDescriptor family) { if (family.getName() == null || family.getName().length <= 0) { - throw new IllegalArgumentException("Family name cannot be null or empty"); - } - if (hasFamily(family.getName())) { - throw new IllegalArgumentException("Family '" + - family.getNameAsString() + "' already exists so cannot be added"); - } - this.families.put(family.getName(), family); - return this; - } - - /** - * Modifies the existing column family. - * @param family HColumnDescriptor of family to update - * @return this (for chained invocation) - */ - public HTableDescriptor modifyFamily(final HColumnDescriptor family) { - if (family.getName() == null || family.getName().length <= 0) { - throw new IllegalArgumentException("Family name cannot be null or empty"); - } - if (!hasFamily(family.getName())) { - throw new IllegalArgumentException("Column family '" + family.getNameAsString() - + "' does not exist"); + throw new NullPointerException("Family name cannot be null or empty"); } this.families.put(family.getName(), family); return this; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java index 5d9c2edf70db..bbe6e9ec4718 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/InvalidFamilyOperationException.java @@ -21,6 +21,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown if a request is table schema modification is requested but * made for an invalid family name. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java deleted file mode 100644 index 6cd52e87f016..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/KeepDeletedCells.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -/** - * Ways to keep cells marked for delete around. - */ -/* - * Don't change the TRUE/FALSE labels below, these have to be called - * this way for backwards compatibility. - */ -public enum KeepDeletedCells { - /** Deleted Cells are not retained. */ - FALSE, - /** - * Deleted Cells are retained until they are removed by other means - * such TTL or VERSIONS. - * If no TTL is specified or no new versions of delete cells are - * written, they are retained forever. - */ - TRUE, - /** - * Deleted Cells are retained until the delete marker expires due to TTL. - * This is useful when TTL is combined with MIN_VERSIONS and one - * wants to keep a minimum number of versions around but at the same - * time remove deleted cells after the TTL. - */ - TTL; -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java index ddd03e8af6ac..a85b1641c62c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MasterNotRunningException.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown if the master is not running */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 3282838b23ec..45a561af96c2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -17,22 +17,13 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -54,8 +45,17 @@ import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ServiceException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Read/write operations on region and assignment information store in diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java index 8975c74a9155..1523ff632ce1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/NotServingRegionException.java @@ -18,12 +18,12 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.Bytes; +import java.io.IOException; + /** * Thrown by a region server if it is sent a request for a region it is not * serving. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java index a5ae44b28be2..33ff8b127805 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/PleaseHoldException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; /** * This exception is thrown by the master when a region server was shut down and diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionException.java index 24ea16c2b39d..13d2f80f57ee 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; /** * Thrown when something happens related to region handling. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java index 234c5aed9107..62e11374d9db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java @@ -160,15 +160,6 @@ public int getStoreUncompressedSizeMB() { return regionLoadPB.getStoreUncompressedSizeMB(); } - /** - * @return the data locality of region in the regionserver. - */ - public float getDataLocality() { - if (regionLoadPB.hasDataLocality()) { - return regionLoadPB.getDataLocality(); - } - return 0.0f; - } /** * @see java.lang.Object#toString() */ @@ -214,8 +205,6 @@ public String toString() { compactionProgressPct); sb = Strings.appendKeyValue(sb, "completeSequenceId", this.getCompleteSequenceId()); - sb = Strings.appendKeyValue(sb, "dataLocality", - this.getDataLocality()); return sb.toString(); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java index 18e5d679fd17..00d22fe15b3f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java @@ -20,19 +20,20 @@ package org.apache.hadoop.hbase; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Strings; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + /** * This class is used for exporting current state of load on a RegionServer. */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableExistsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableExistsException.java index 00cabbfaf132..9590e4be8b0b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableExistsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableExistsException.java @@ -16,6 +16,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; /** * Thrown when a table exists but should not diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java index 9b5f728afb6b..ea707bf31763 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotDisabledException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java index 0f78ee60d8d8..210b875a4b5f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotEnabledException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java index 8ac5e20d4080..2433a14e78db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/TableNotFoundException.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; /** Thrown when a table can not be located */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java index 37d5ccc450d9..94dec2417855 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/YouAreDeadException.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * This exception is thrown by the master when a region server reports and is * already being processed as dead. This can happen when a region server loses diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java index 422a6594700d..7aebf33af459 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ZooKeeperConnectionException.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown if the client can't connect to zookeeper */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java index 5743fd5eac09..18be1d9822f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java @@ -18,8 +18,9 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; /** * A Get, Put, Increment, Append, or Delete associated with it's region. Used internally by diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index a08cf79f1c8b..6d06efee19e3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -18,12 +18,7 @@ */ package org.apache.hadoop.hbase.client; -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - +import com.google.protobuf.ServiceException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.ClusterStatus; @@ -39,9 +34,9 @@ import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; @@ -55,7 +50,11 @@ import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; import org.apache.hadoop.hbase.util.Pair; -import com.google.protobuf.ServiceException; +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; /** * The administrative API for HBase. Obtain an instance from an {@link Connection#getAdmin()} and diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java index 9e3be3e69a30..6556f8bfd66d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java @@ -23,11 +23,11 @@ import java.util.NavigableMap; import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.CellVisibility; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Attributes.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Attributes.java index 78d33981c5c6..dbf9da94ff15 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Attributes.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Attributes.java @@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.client; -import java.util.Map; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.util.Map; + @InterfaceAudience.Public @InterfaceStability.Stable public interface Attributes { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientIdGenerator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientIdGenerator.java index ac6c82e8f7c6..652ae9984d49 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientIdGenerator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientIdGenerator.java @@ -20,6 +20,11 @@ import java.io.IOException; import java.lang.management.ManagementFactory; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Enumeration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 05202c8f77f5..9d5924288a69 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -34,7 +35,7 @@ import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownScannerException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java index 86ff4241b159..2cab8300478a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallReversedScanner.java @@ -20,20 +20,20 @@ package org.apache.hadoop.hbase.client; -import java.io.IOException; -import java.util.concurrent.ExecutorService; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.util.Bytes; +import java.io.IOException; +import java.util.concurrent.ExecutorService; + /** * Client scanner for small reversed scan. Generally, only one RPC is called to fetch the * scan results, unless the results cross multiple regions or the row count of diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java index 90fb1c2c2093..478ba768f29c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientSmallScanner.java @@ -24,12 +24,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnection.java index 398ecadcf760..ad67f63a0896 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnection.java @@ -21,14 +21,13 @@ import java.io.IOException; import java.util.List; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService; @@ -271,14 +270,4 @@ MasterKeepAliveConnection getKeepAliveMasterService() * @return Default AsyncProcess associated with this connection. */ AsyncProcess getAsyncProcess(); - - /** - * Returns a new RpcRetryingCallerFactory from the given {@link Configuration}. - * This RpcRetryingCallerFactory lets the users create {@link RpcRetryingCaller}s which can be - * intercepted with the configured {@link RetryingCallerInterceptor} - * @param conf - * @return RpcRetryingCallerFactory - */ - RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf); } - diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java index 575623276dec..475ae019ff41 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java @@ -30,30 +30,29 @@ import io.netty.channel.socket.DatagramChannel; import io.netty.channel.socket.DatagramPacket; import io.netty.channel.socket.nio.NioDatagramChannel; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.ExceptionUtil; import org.apache.hadoop.hbase.util.Threads; +import java.io.Closeable; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + /** * A class that receives the cluster status, and provide it as a set of service to the client. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java index 55237be5ba54..92b3f042dcf8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.util.concurrent.ExecutorService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; /** * A cluster connection encapsulating lower level individual connections to actual servers and diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionAdapter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionAdapter.java index 927b56c1d67d..9ffdddb7d581 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionAdapter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionAdapter.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; @@ -29,7 +30,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; @@ -440,9 +440,4 @@ public NonceGenerator getNonceGenerator() { public AsyncProcess getAsyncProcess() { return wrappedConnection.getAsyncProcess(); } - - @Override - public RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf) { - return wrappedConnection.getNewRpcRetryingCallerFactory(conf); - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java index 374ce280c69b..4af674014894 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java @@ -22,11 +22,11 @@ import java.lang.reflect.Constructor; import java.util.concurrent.ExecutorService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index b3a6295ea55d..c7270baae419 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -42,6 +42,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -59,7 +60,6 @@ import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase; @@ -583,8 +583,6 @@ static class HConnectionImplementation implements ClusterConnection, Closeable { private RpcControllerFactory rpcControllerFactory; - private final RetryingCallerInterceptor interceptor; - /** * Cluster registry of basic info such as clusterid and meta region location. */ @@ -615,6 +613,7 @@ static class HConnectionImplementation implements ClusterConnection, Closeable { retrieveClusterId(); this.rpcClient = new RpcClient(this.conf, this.clusterId); + this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf); this.rpcControllerFactory = RpcControllerFactory.instantiate(conf); // Do we publish the status? @@ -665,8 +664,6 @@ protected HConnectionImplementation(Configuration conf) { this.nonceGenerator = new NoNonceGenerator(); } this.asyncProcess = createAsyncProcess(this.conf); - this.interceptor = (new RetryingCallerInterceptorFactory(conf)).build(); - this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf, interceptor); } @Override @@ -2512,11 +2509,6 @@ public TableState getTableState(TableName tableName) throws IOException { master.close(); } } - - @Override - public RpcRetryingCallerFactory getNewRpcRetryingCallerFactory(Configuration conf) { - return RpcRetryingCallerFactory.instantiate(conf, this.interceptor); - } } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java index e26ae4826e86..f38c4e07de30 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java @@ -22,11 +22,11 @@ import java.util.concurrent.ExecutorService; import org.apache.commons.logging.Log; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.security.User; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java index ea76d0dc3c38..0533b313deae 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -26,12 +26,12 @@ import java.util.NavigableMap; import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.CellVisibility; import org.apache.hadoop.hbase.util.Bytes; @@ -43,15 +43,15 @@ * to delete. To further define the scope of what to delete, perform * additional methods as outlined below. *

- * To delete specific families, execute {@link #addFamily(byte[]) deleteFamily} + * To delete specific families, execute {@link #deleteFamily(byte[]) deleteFamily} * for each family to delete. *

* To delete multiple versions of specific columns, execute - * {@link #addColumns(byte[], byte[]) deleteColumns} + * {@link #deleteColumns(byte[], byte[]) deleteColumns} * for each column to delete. *

* To delete specific versions of specific columns, execute - * {@link #addColumn(byte[], byte[], long) deleteColumn} + * {@link #deleteColumn(byte[], byte[], long) deleteColumn} * for each column version to delete. *

* Specifying timestamps, deleteFamily and deleteColumns will delete all @@ -185,22 +185,8 @@ public Delete addDeleteMarker(Cell kv) throws IOException { * specified family. * @param family family name * @return this for invocation chaining - * @deprecated Since 1.0.0. Use {@link #addFamily(byte[])} */ - @Deprecated public Delete deleteFamily(byte [] family) { - return addFamily(family); - } - - /** - * Delete all versions of all columns of the specified family. - *

- * Overrides previous calls to deleteColumn and deleteColumns for the - * specified family. - * @param family family name - * @return this for invocation chaining - */ - public Delete addFamily(final byte [] family) { this.deleteFamily(family, this.ts); return this; } @@ -214,24 +200,9 @@ public Delete addFamily(final byte [] family) { * @param family family name * @param timestamp maximum version timestamp * @return this for invocation chaining - * @deprecated Since 1.0.0. Use {@link #addFamily(byte[], long)} */ - @Deprecated + @SuppressWarnings("unchecked") public Delete deleteFamily(byte [] family, long timestamp) { - return addFamily(family, timestamp); - } - - /** - * Delete all columns of the specified family with a timestamp less than - * or equal to the specified timestamp. - *

- * Overrides previous calls to deleteColumn and deleteColumns for the - * specified family. - * @param family family name - * @param timestamp maximum version timestamp - * @return this for invocation chaining - */ - public Delete addFamily(final byte [] family, final long timestamp) { if (timestamp < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp); } @@ -255,19 +226,6 @@ public Delete addFamily(final byte [] family, final long timestamp) { * @return this for invocation chaining */ public Delete deleteFamilyVersion(byte [] family, long timestamp) { - return addFamilyVersion(family, timestamp); - } - - /** - * Delete all columns of the specified family with a timestamp equal to - * the specified timestamp. - * @param family family name - * @param timestamp version timestamp - * @return this for invocation chaining - * @deprecated Since hbase-1.0.0. Use {@link #addFamilyVersion(byte[], long)} - */ - @Deprecated - public Delete addFamilyVersion(final byte [] family, final long timestamp) { List list = familyMap.get(family); if(list == null) { list = new ArrayList(); @@ -278,17 +236,6 @@ public Delete addFamilyVersion(final byte [] family, final long timestamp) { return this; } - /** - * Delete all versions of the specified column. - * @param family family name - * @param qualifier column qualifier - * @return this for invocation chaining - * @deprecated Since hbase-1.0.0. Use {@link #addColumns(byte[], byte[])} - */ - @Deprecated - public Delete deleteColumns(byte [] family, byte [] qualifier) { - return addColumns(family, qualifier); - } /** * Delete all versions of the specified column. @@ -296,8 +243,8 @@ public Delete deleteColumns(byte [] family, byte [] qualifier) { * @param qualifier column qualifier * @return this for invocation chaining */ - public Delete addColumns(final byte [] family, final byte [] qualifier) { - addColumns(family, qualifier, this.ts); + public Delete deleteColumns(byte [] family, byte [] qualifier) { + this.deleteColumns(family, qualifier, this.ts); return this; } @@ -308,22 +255,9 @@ public Delete addColumns(final byte [] family, final byte [] qualifier) { * @param qualifier column qualifier * @param timestamp maximum version timestamp * @return this for invocation chaining - * @deprecated Since hbase-1.0.0. Use {@link #addColumns(byte[], byte[], long)} */ - @Deprecated + @SuppressWarnings("unchecked") public Delete deleteColumns(byte [] family, byte [] qualifier, long timestamp) { - return addColumns(family, qualifier, timestamp); - } - - /** - * Delete all versions of the specified column with a timestamp less than - * or equal to the specified timestamp. - * @param family family name - * @param qualifier column qualifier - * @param timestamp maximum version timestamp - * @return this for invocation chaining - */ - public Delete addColumns(final byte [] family, final byte [] qualifier, final long timestamp) { if (timestamp < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp); } @@ -345,23 +279,8 @@ public Delete addColumns(final byte [] family, final byte [] qualifier, final lo * @param family family name * @param qualifier column qualifier * @return this for invocation chaining - * @deprecated Since hbase-1.0.0. Use {@link #addColumn(byte[], byte[])} */ - @Deprecated public Delete deleteColumn(byte [] family, byte [] qualifier) { - return addColumn(family, qualifier); - } - - /** - * Delete the latest version of the specified column. - * This is an expensive call in that on the server-side, it first does a - * get to find the latest versions timestamp. Then it adds a delete using - * the fetched cells timestamp. - * @param family family name - * @param qualifier column qualifier - * @return this for invocation chaining - */ - public Delete addColumn(final byte [] family, final byte [] qualifier) { this.deleteColumn(family, qualifier, this.ts); return this; } @@ -372,21 +291,9 @@ public Delete addColumn(final byte [] family, final byte [] qualifier) { * @param qualifier column qualifier * @param timestamp version timestamp * @return this for invocation chaining - * @deprecated Since hbase-1.0.0. Use {@link #addColumn(byte[], byte[], long)} */ - @Deprecated + @SuppressWarnings("unchecked") public Delete deleteColumn(byte [] family, byte [] qualifier, long timestamp) { - return addColumn(family, qualifier, timestamp); - } - - /** - * Delete the specified version of the specified column. - * @param family family name - * @param qualifier column qualifier - * @param timestamp version timestamp - * @return this for invocation chaining - */ - public Delete addColumn(byte [] family, byte [] qualifier, long timestamp) { if (timestamp < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FailureInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FailureInfo.java deleted file mode 100644 index 9d685b8566dd..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FailureInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.classification.InterfaceAudience; - -/** - * Keeps track of repeated failures to any region server. Multiple threads manipulate the contents - * of this thread. - * - * Access to the members is guarded by the concurrent nature of the members inherently. - * - */ -@InterfaceAudience.Private -class FailureInfo { - // The number of consecutive failures. - public final AtomicLong numConsecutiveFailures = new AtomicLong(); - // The time when the server started to become unresponsive - // Once set, this would never be updated. - public final long timeOfFirstFailureMilliSec; - // The time when the client last tried to contact the server. - // This is only updated by one client at a time - public volatile long timeOfLatestAttemptMilliSec; - // Used to keep track of concurrent attempts to contact the server. - // In Fast fail mode, we want just one client thread to try to connect - // the rest of the client threads will fail fast. - public final AtomicBoolean exclusivelyRetringInspiteOfFastFail = new AtomicBoolean( - false); - - @Override - public String toString() { - return "FailureInfo: numConsecutiveFailures = " - + numConsecutiveFailures + " timeOfFirstFailureMilliSec = " - + timeOfFirstFailureMilliSec + " timeOfLatestAttemptMilliSec = " - + timeOfLatestAttemptMilliSec - + " exclusivelyRetringInspiteOfFastFail = " - + exclusivelyRetringInspiteOfFastFail.get(); - } - - FailureInfo(long firstFailureTime) { - this.timeOfFirstFailureMilliSec = firstFailureTime; - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java deleted file mode 100644 index 9eb56bca1011..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import org.apache.commons.lang.mutable.MutableBoolean; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -@InterfaceAudience.Private -class FastFailInterceptorContext extends - RetryingCallerInterceptorContext { - - // The variable that indicates whether we were able to connect with the server - // in the last run - private MutableBoolean couldNotCommunicateWithServer = new MutableBoolean( - false); - - // The variable which indicates whether this was a retry or the first time - private boolean didTry = false; - - // The failure info that is associated with the machine which we are trying to - // contact as part of this attempt. - private FailureInfo fInfo = null; - - // Variable indicating that the thread that is currently executing the - // operation is in a mode where it would retry instead of failing fast, so - // that we can figure out whether making contact with the server is - // possible or not. - private boolean retryDespiteFastFailMode = false; - - // The server that would be contacted to successfully complete this operation. - private ServerName server; - - // The number of the retry we are currenty doing. - private int tries; - - public MutableBoolean getCouldNotCommunicateWithServer() { - return couldNotCommunicateWithServer; - } - - public FailureInfo getFailureInfo() { - return fInfo; - } - - public ServerName getServer() { - return server; - } - - public int getTries() { - return tries; - } - - public boolean didTry() { - return didTry; - } - - public boolean isRetryDespiteFastFailMode() { - return retryDespiteFastFailMode; - } - - public void setCouldNotCommunicateWithServer( - MutableBoolean couldNotCommunicateWithServer) { - this.couldNotCommunicateWithServer = couldNotCommunicateWithServer; - } - - public void setDidTry(boolean didTry) { - this.didTry = didTry; - } - - public void setFailureInfo(FailureInfo fInfo) { - this.fInfo = fInfo; - } - - public void setRetryDespiteFastFailMode(boolean retryDespiteFastFailMode) { - this.retryDespiteFastFailMode = retryDespiteFastFailMode; - } - - public void setServer(ServerName server) { - this.server = server; - } - - public void setTries(int tries) { - this.tries = tries; - } - - public void clear() { - server = null; - fInfo = null; - didTry = false; - couldNotCommunicateWithServer.setValue(false); - retryDespiteFastFailMode = false; - tries = 0; - } - - public FastFailInterceptorContext prepare(RetryingCallable callable) { - return prepare(callable, 0); - } - - public FastFailInterceptorContext prepare(RetryingCallable callable, - int tries) { - if (callable instanceof RegionServerCallable) { - RegionServerCallable retryingCallable = (RegionServerCallable) callable; - server = retryingCallable.getLocation().getServerName(); - } - this.tries = tries; - return this; - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java index 701cd9c1cb77..1d3310e0832c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java @@ -31,9 +31,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.security.access.Permission; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 75dd72ca5d6e..7136f72e1c28 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -68,9 +68,11 @@ import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel; +import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; +import org.apache.hadoop.hbase.protobuf.ResponseConverter; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse; @@ -83,6 +85,9 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationRequest; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription; @@ -106,6 +111,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest; @@ -661,24 +668,47 @@ public Void call(int callTimeout) throws ServiceException { // Wait until all regions deleted for (int tries = 0; tries < (this.numRetries * this.retryLongerMultiplier); tries++) { try { - // Find whether all regions are deleted. - List regionLations = - MetaScanner.listTableRegionLocations(conf, connection, tableName); + HRegionLocation firstMetaServer = getFirstMetaServerForTable(tableName); + Scan scan = MetaTableAccessor.getScanForTableName(tableName); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + ScanRequest request = RequestConverter.buildScanRequest( + firstMetaServer.getRegionInfo().getRegionName(), scan, 1, true); + Result[] values = null; + // Get a batch at a time. + ClientService.BlockingInterface server = connection.getClient(firstMetaServer + .getServerName()); + PayloadCarryingRpcController controller = new PayloadCarryingRpcController(); + try { + controller.setPriority(tableName); + ScanResponse response = server.scan(controller, request); + values = ResponseConverter.getResults(controller.cellScanner(), response); + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } // let us wait until hbase:meta table is updated and // HMaster removes the table from its HTableDescriptors - if (regionLations == null || regionLations.size() == 0) { - HTableDescriptor htd = getTableDescriptorByTableName(tableName); - - if (htd == null) { - // table could not be found in master - we are done. - tableExists = false; + if (values == null || values.length == 0) { + tableExists = false; + GetTableDescriptorsResponse htds; + MasterKeepAliveConnection master = connection.getKeepAliveMasterService(); + try { + GetTableDescriptorsRequest req = + RequestConverter.buildGetTableDescriptorsRequest(tableName); + htds = master.getTableDescriptors(null, req); + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } finally { + master.close(); + } + tableExists = !htds.getTableSchemaList().isEmpty(); + if (!tableExists) { break; } } } catch (IOException ex) { failures++; - if(failures >= numRetries - 1) { // no more tries left + if(failures == numRetries - 1) { // no more tries left if (ex instanceof RemoteException) { throw ((RemoteException) ex).unwrapRemoteException(); } else { @@ -2589,27 +2619,6 @@ public HTableDescriptor[] getTableDescriptorsByTableName(List tableNa return this.connection.getHTableDescriptorsByTableName(tableNames); } - /** - * Get tableDescriptor - * @param tableName one table name - * @return HTD the HTableDescriptor or null if the table not exists - * @throws IOException if a remote or network exception occurs - */ - private HTableDescriptor getTableDescriptorByTableName(TableName tableName) - throws IOException { - List tableNames = new ArrayList(1); - tableNames.add(tableName); - - HTableDescriptor[] htdl = getTableDescriptorsByTableName(tableNames); - - if (htdl == null || htdl.length == 0) { - return null; - } - else { - return htdl[0]; - } - } - /** * Get tableDescriptors * @param names List of table names diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectable.java index c4f7b103eb0b..8863bc149d4e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectable.java @@ -21,8 +21,8 @@ import java.io.IOException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; /** * This class makes it convenient for one to execute a command in the context diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java index b5328bf901c5..fa62923c0136 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnection.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.concurrent.ExecutorService; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; @@ -29,8 +31,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionKey.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionKey.java index 0c2018189c5d..45c9bbbfc1b2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionKey.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionKey.java @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; @@ -48,8 +47,7 @@ class HConnectionKey { HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.HBASE_META_SCANNER_CACHING, HConstants.HBASE_CLIENT_INSTANCE_ID, - HConstants.RPC_CODEC_CONF_KEY, - RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY}; + HConstants.RPC_CODEC_CONF_KEY }; private Map properties; private String username; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index e81fad0d9213..b979f877a58d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -20,12 +20,11 @@ import java.io.IOException; import java.util.concurrent.ExecutorService; - import org.apache.commons.logging.Log; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.security.User; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index c3a94e33e11a..a1650c1d139d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -38,6 +38,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -49,8 +51,6 @@ import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback; @@ -358,7 +358,7 @@ private void finishSetup() throws IOException { this.retries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - this.rpcCallerFactory = connection.getNewRpcRetryingCallerFactory(configuration); + this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(configuration); this.rpcControllerFactory = RpcControllerFactory.instantiate(configuration); // puts need to track errors globally due to how the APIs currently work. ap = new AsyncProcess(connection, configuration, pool, rpcCallerFactory, true, rpcControllerFactory); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java index 6970333557f5..d053e66a38e7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableFactory.java @@ -18,12 +18,12 @@ */ package org.apache.hadoop.hbase.client; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; /** * Factory for creating HTable instances. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java index b02caa8ca97d..f3b03bf68a63 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTableInterfaceFactory.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase.client; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java index b12ee02ebfef..6fc3961447b3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java @@ -25,11 +25,11 @@ import java.util.TreeMap; import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.CellVisibility; @@ -95,6 +95,7 @@ public Increment(Increment i) { * @return this * @throws java.io.IOException e */ + @SuppressWarnings("unchecked") public Increment add(Cell cell) throws IOException{ byte [] family = CellUtil.cloneFamily(cell); List list = getCellList(family); @@ -120,6 +121,7 @@ public Increment add(Cell cell) throws IOException{ * @param amount amount to increment by * @return the Increment object */ + @SuppressWarnings("unchecked") public Increment addColumn(byte [] family, byte [] qualifier, long amount) { if (family == null) { throw new IllegalArgumentException("family cannot be null"); @@ -237,7 +239,7 @@ public String toString() { } else { moreThanOneB = true; } - sb.append(CellUtil.getCellKeyAsString(cell) + "+=" + + sb.append(CellUtil.getCellKey(cell) + "+=" + Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } sb.append("}"); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java index 4ecddb59b66e..aaaf54524b16 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java @@ -25,16 +25,15 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java index 5312dfb4135b..ede542a19bd8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -37,7 +38,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ExceptionUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java index 8f1dc4d0e1bb..eefe40d5f159 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java @@ -24,8 +24,8 @@ import java.util.Set; import java.util.TreeMap; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java index 089ccff1f844..9f03d719705f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiResponse.java @@ -19,12 +19,16 @@ package org.apache.hadoop.hbase.client; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; /** * A container for Result objects, grouped by regionName. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java index b7f7d1b92864..a75d3b0ac1b4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java @@ -28,6 +28,8 @@ import java.util.TreeMap; import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; @@ -35,8 +37,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryableCallerInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryableCallerInterceptor.java deleted file mode 100644 index f3f9168a4a04..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryableCallerInterceptor.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException; - -/** - * Class that acts as a NoOpInterceptor. This class is used in case the - * {@link RetryingCallerInterceptor} was not configured correctly or an - * {@link RetryingCallerInterceptor} was never configured in the first place. - * - */ -@InterfaceAudience.Private -class NoOpRetryableCallerInterceptor extends RetryingCallerInterceptor { - private static final RetryingCallerInterceptorContext NO_OP_CONTEXT = - new NoOpRetryingInterceptorContext(); - - public NoOpRetryableCallerInterceptor() { - } - - public NoOpRetryableCallerInterceptor(Configuration conf) { - super(); - } - - @Override - public void intercept( - RetryingCallerInterceptorContext abstractRetryingCallerInterceptorContext) - throws PreemptiveFastFailException { - } - - @Override - public void handleFailure(RetryingCallerInterceptorContext context, - Throwable t) throws IOException { - } - - @Override - public void updateFailureInfo(RetryingCallerInterceptorContext context) { - } - - @Override - public RetryingCallerInterceptorContext createEmptyContext() { - return NO_OP_CONTEXT; - } - - @Override - public String toString() { - return "NoOpRetryableCallerInterceptor"; - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java deleted file mode 100644 index 1ccf43c096e6..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -@InterfaceAudience.Private -class NoOpRetryingInterceptorContext extends RetryingCallerInterceptorContext { - - @Override - public void clear() { - // Do Nothing - } - - @Override - public RetryingCallerInterceptorContext prepare( - RetryingCallable callable) { - // Do Nothing - return this; - } - - @Override - public RetryingCallerInterceptorContext prepare( - RetryingCallable callable, int tries) { - // Do Nothing - return this; - } - -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java index e2e06b9dd6ac..672f7209d6f0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoServerForRegionException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.RegionException; /** * Thrown when no region server can be found for a region diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java index 9fdd5775ef78..d9d54ea55fc0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java @@ -19,15 +19,15 @@ package org.apache.hadoop.hbase.client; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + @InterfaceAudience.Public @InterfaceStability.Evolving public abstract class OperationWithAttributes extends Operation implements Attributes { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PerClientRandomNonceGenerator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PerClientRandomNonceGenerator.java index 875e1f6d9db3..7ac4546f641b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PerClientRandomNonceGenerator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PerClientRandomNonceGenerator.java @@ -21,8 +21,8 @@ import java.util.Arrays; import java.util.Random; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; /** * NonceGenerator implementation that uses client ID hash + random int as nonce group, diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java deleted file mode 100644 index 425612068482..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/PreemptiveFastFailInterceptor.java +++ /dev/null @@ -1,405 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.io.EOFException; -import java.io.IOException; -import java.io.SyncFailedException; -import java.lang.reflect.UndeclaredThrowableException; -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.nio.channels.ClosedChannelException; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.lang.mutable.MutableBoolean; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.exceptions.ConnectionClosingException; -import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException; -import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.hadoop.ipc.RemoteException; - -/** - * - * The concrete {@link RetryingCallerInterceptor} class that implements the preemptive fast fail - * feature. - * - * The motivation is as follows : - * In case where a large number of clients try and talk to a particular region server in hbase, if - * the region server goes down due to network problems, we might end up in a scenario where - * the clients would go into a state where they all start to retry. - * This behavior will set off many of the threads in pretty much the same path and they all would be - * sleeping giving rise to a state where the client either needs to create more threads to send new - * requests to other hbase machines or block because the client cannot create anymore threads. - * - * In most cases the clients might prefer to have a bound on the number of threads that are created - * in order to send requests to hbase. This would mostly result in the client thread starvation. - * - * To circumvent this problem, the approach that is being taken here under is to let 1 of the many - * threads who are trying to contact the regionserver with connection problems and let the other - * threads get a {@link PreemptiveFastFailException} so that they can move on and take other - * requests. - * - * This would give the client more flexibility on the kind of action he would want to take in cases - * where the regionserver is down. He can either discard the requests and send a nack upstream - * faster or have an application level retry or buffer the requests up so as to send them down to - * hbase later. - * - */ -@InterfaceAudience.Private -class PreemptiveFastFailInterceptor extends RetryingCallerInterceptor { - - public static final Log LOG = LogFactory - .getLog(PreemptiveFastFailInterceptor.class); - - // amount of time to wait before we consider a server to be in fast fail - // mode - protected final long fastFailThresholdMilliSec; - - // Keeps track of failures when we cannot talk to a server. Helps in - // fast failing clients if the server is down for a long time. - protected final ConcurrentMap repeatedFailuresMap = - new ConcurrentHashMap(); - - // We populate repeatedFailuresMap every time there is a failure. So, to - // keep it from growing unbounded, we garbage collect the failure information - // every cleanupInterval. - protected final long failureMapCleanupIntervalMilliSec; - - protected volatile long lastFailureMapCleanupTimeMilliSec; - - // clear failure Info. Used to clean out all entries. - // A safety valve, in case the client does not exit the - // fast fail mode for any reason. - private long fastFailClearingTimeMilliSec; - - private final ThreadLocal threadRetryingInFastFailMode = - new ThreadLocal(); - - public PreemptiveFastFailInterceptor(Configuration conf) { - this.fastFailThresholdMilliSec = conf.getLong( - HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, - HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT); - this.failureMapCleanupIntervalMilliSec = conf.getLong( - HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS, - HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT); - lastFailureMapCleanupTimeMilliSec = EnvironmentEdgeManager.currentTime(); - } - - public void intercept(FastFailInterceptorContext context) - throws PreemptiveFastFailException { - context.setFailureInfo(repeatedFailuresMap.get(context.getServer())); - if (inFastFailMode(context.getServer()) && !currentThreadInFastFailMode()) { - // In Fast-fail mode, all but one thread will fast fail. Check - // if we are that one chosen thread. - context.setRetryDespiteFastFailMode(shouldRetryInspiteOfFastFail(context - .getFailureInfo())); - if (!context.isRetryDespiteFastFailMode()) { // we don't have to retry - LOG.debug("Throwing PFFE : " + context.getFailureInfo() + " tries : " - + context.getTries()); - throw new PreemptiveFastFailException( - context.getFailureInfo().numConsecutiveFailures.get(), - context.getFailureInfo().timeOfFirstFailureMilliSec, - context.getFailureInfo().timeOfLatestAttemptMilliSec, context.getServer()); - } - } - context.setDidTry(true); - } - - public void handleFailure(FastFailInterceptorContext context, - Throwable t) throws IOException { - handleThrowable(t, context.getServer(), - context.getCouldNotCommunicateWithServer()); - } - - public void updateFailureInfo(FastFailInterceptorContext context) { - updateFailureInfoForServer(context.getServer(), context.getFailureInfo(), - context.didTry(), context.getCouldNotCommunicateWithServer() - .booleanValue(), context.isRetryDespiteFastFailMode()); - } - - /** - * Handles failures encountered when communicating with a server. - * - * Updates the FailureInfo in repeatedFailuresMap to reflect the failure. - * Throws RepeatedConnectException if the client is in Fast fail mode. - * - * @param serverName - * @param t - * - the throwable to be handled. - * @throws PreemptiveFastFailException - */ - private void handleFailureToServer(ServerName serverName, Throwable t) { - if (serverName == null || t == null) { - return; - } - long currentTime = EnvironmentEdgeManager.currentTime(); - FailureInfo fInfo = repeatedFailuresMap.get(serverName); - if (fInfo == null) { - fInfo = new FailureInfo(currentTime); - FailureInfo oldfInfo = repeatedFailuresMap.putIfAbsent(serverName, fInfo); - - if (oldfInfo != null) { - fInfo = oldfInfo; - } - } - fInfo.timeOfLatestAttemptMilliSec = currentTime; - fInfo.numConsecutiveFailures.incrementAndGet(); - } - - public void handleThrowable(Throwable t1, ServerName serverName, - MutableBoolean couldNotCommunicateWithServer) throws IOException { - Throwable t2 = translateException(t1); - boolean isLocalException = !(t2 instanceof RemoteException); - if (isLocalException && isConnectionException(t2)) { - couldNotCommunicateWithServer.setValue(true); - handleFailureToServer(serverName, t2); - } - } - - private Throwable translateException(Throwable t) throws IOException { - if (t instanceof NoSuchMethodError) { - // We probably can't recover from this exception by retrying. - LOG.error(t); - throw (NoSuchMethodError) t; - } - - if (t instanceof NullPointerException) { - // The same here. This is probably a bug. - LOG.error(t.getMessage(), t); - throw (NullPointerException) t; - } - - if (t instanceof UndeclaredThrowableException) { - t = t.getCause(); - } - if (t instanceof RemoteException) { - t = ((RemoteException) t).unwrapRemoteException(); - } - if (t instanceof DoNotRetryIOException) { - throw (DoNotRetryIOException) t; - } - if (t instanceof Error) { - throw (Error) t; - } - return t; - } - - /** - * Check if the exception is something that indicates that we cannot - * contact/communicate with the server. - * - * @param e - * @return true when exception indicates that the client wasn't able to make contact with server - */ - private boolean isConnectionException(Throwable e) { - if (e == null) - return false; - // This list covers most connectivity exceptions but not all. - // For example, in SocketOutputStream a plain IOException is thrown - // at times when the channel is closed. - return (e instanceof SocketTimeoutException - || e instanceof ConnectException || e instanceof ClosedChannelException - || e instanceof SyncFailedException || e instanceof EOFException - || e instanceof TimeoutException - || e instanceof ConnectionClosingException || e instanceof FailedServerException); - } - - /** - * Occasionally cleans up unused information in repeatedFailuresMap. - * - * repeatedFailuresMap stores the failure information for all remote hosts - * that had failures. In order to avoid these from growing indefinitely, - * occassionallyCleanupFailureInformation() will clear these up once every - * cleanupInterval ms. - */ - protected void occasionallyCleanupFailureInformation() { - long now = System.currentTimeMillis(); - if (!(now > lastFailureMapCleanupTimeMilliSec - + failureMapCleanupIntervalMilliSec)) - return; - - // remove entries that haven't been attempted in a while - // No synchronization needed. It is okay if multiple threads try to - // remove the entry again and again from a concurrent hash map. - StringBuilder sb = new StringBuilder(); - for (Entry entry : repeatedFailuresMap.entrySet()) { - if (now > entry.getValue().timeOfLatestAttemptMilliSec - + failureMapCleanupIntervalMilliSec) { // no recent failures - repeatedFailuresMap.remove(entry.getKey()); - } else if (now > entry.getValue().timeOfFirstFailureMilliSec - + this.fastFailClearingTimeMilliSec) { // been failing for a long - // time - LOG.error(entry.getKey() - + " been failing for a long time. clearing out." - + entry.getValue().toString()); - repeatedFailuresMap.remove(entry.getKey()); - } else { - sb.append(entry.getKey().toString()).append(" failing ") - .append(entry.getValue().toString()).append("\n"); - } - } - if (sb.length() > 0) { - LOG.warn("Preemptive failure enabled for : " + sb.toString()); - } - lastFailureMapCleanupTimeMilliSec = now; - } - - /** - * Checks to see if we are in the Fast fail mode for requests to the server. - * - * If a client is unable to contact a server for more than - * fastFailThresholdMilliSec the client will get into fast fail mode. - * - * @param server - * @return true if the client is in fast fail mode for the server. - */ - private boolean inFastFailMode(ServerName server) { - FailureInfo fInfo = repeatedFailuresMap.get(server); - // if fInfo is null --> The server is considered good. - // If the server is bad, wait long enough to believe that the server is - // down. - return (fInfo != null && - EnvironmentEdgeManager.currentTime() > - (fInfo.timeOfFirstFailureMilliSec + this.fastFailThresholdMilliSec)); - } - - /** - * Checks to see if the current thread is already in FastFail mode for *some* - * server. - * - * @return true, if the thread is already in FF mode. - */ - private boolean currentThreadInFastFailMode() { - return (this.threadRetryingInFastFailMode.get() != null && (this.threadRetryingInFastFailMode - .get().booleanValue() == true)); - } - - /** - * Check to see if the client should try to connnect to the server, inspite of - * knowing that it is in the fast fail mode. - * - * The idea here is that we want just one client thread to be actively trying - * to reconnect, while all the other threads trying to reach the server will - * short circuit. - * - * @param fInfo - * @return true if the client should try to connect to the server. - */ - protected boolean shouldRetryInspiteOfFastFail(FailureInfo fInfo) { - // We believe that the server is down, But, we want to have just one - // client - // actively trying to connect. If we are the chosen one, we will retry - // and not throw an exception. - if (fInfo != null - && fInfo.exclusivelyRetringInspiteOfFastFail.compareAndSet(false, true)) { - MutableBoolean threadAlreadyInFF = this.threadRetryingInFastFailMode - .get(); - if (threadAlreadyInFF == null) { - threadAlreadyInFF = new MutableBoolean(); - this.threadRetryingInFastFailMode.set(threadAlreadyInFF); - } - threadAlreadyInFF.setValue(true); - return true; - } else { - return false; - } - } - - /** - * - * This function updates the Failure info for a particular server after the - * attempt to - * - * @param server - * @param fInfo - * @param couldNotCommunicate - * @param retryDespiteFastFailMode - */ - private void updateFailureInfoForServer(ServerName server, - FailureInfo fInfo, boolean didTry, boolean couldNotCommunicate, - boolean retryDespiteFastFailMode) { - if (server == null || fInfo == null || didTry == false) - return; - - // If we were able to connect to the server, reset the failure - // information. - if (couldNotCommunicate == false) { - LOG.info("Clearing out PFFE for server " + server.getServerName()); - repeatedFailuresMap.remove(server); - } else { - // update time of last attempt - long currentTime = System.currentTimeMillis(); - fInfo.timeOfLatestAttemptMilliSec = currentTime; - - // Release the lock if we were retrying inspite of FastFail - if (retryDespiteFastFailMode) { - fInfo.exclusivelyRetringInspiteOfFastFail.set(false); - threadRetryingInFastFailMode.get().setValue(false); - } - } - - occasionallyCleanupFailureInformation(); - } - - @Override - public void intercept(RetryingCallerInterceptorContext context) - throws PreemptiveFastFailException { - if (context instanceof FastFailInterceptorContext) { - intercept((FastFailInterceptorContext) context); - } - } - - @Override - public void handleFailure(RetryingCallerInterceptorContext context, - Throwable t) throws IOException { - if (context instanceof FastFailInterceptorContext) { - handleFailure((FastFailInterceptorContext) context, t); - } - } - - @Override - public void updateFailureInfo(RetryingCallerInterceptorContext context) { - if (context instanceof FastFailInterceptorContext) { - updateFailureInfo((FastFailInterceptorContext) context); - } - } - - @Override - public RetryingCallerInterceptorContext createEmptyContext() { - return new FastFailInterceptorContext(); - } - - protected boolean isServerInFailureMap(ServerName serverName) { - return this.repeatedFailuresMap.containsKey(serverName); - } - - @Override - public String toString() { - return "PreemptiveFastFailInterceptor"; - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java index f0d3a72dda8c..7a9562ed5e9e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java @@ -28,13 +28,13 @@ import java.util.TreeMap; import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.CellVisibility; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java index 793c3a1f4929..747bdc3cf983 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.Authorizations; import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; - import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorServiceExec.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorServiceExec.java index ad1d2a19d6d6..8e9f380333a8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorServiceExec.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionCoprocessorServiceExec.java @@ -19,13 +19,14 @@ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.Bytes; - import com.google.common.base.Objects; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Message; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.util.Bytes; + /** * Represents a coprocessor service method execution against a single region. While coprocessor diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java index 8168fe1033e7..677b3c5ac49d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java @@ -18,16 +18,16 @@ */ package org.apache.hadoop.hbase.client; -import java.io.Closeable; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Pair; +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + /** * Used to view region location information for a single HBase table. * Obtain an instance from an {@link HConnection}. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java index d6cceb9cd98b..6c1d1cdcba6d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionOfflineException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.RegionException; /** Thrown when a table can not be located */ @InterfaceAudience.Public diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java index 91d1f9b1d6c0..01c5234f7668 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java @@ -21,8 +21,8 @@ import java.util.Collection; import java.util.Iterator; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; /** * Utility methods which contain the logic for regions and replicas. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java index b661ea4e4137..888fd9391441 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java @@ -25,11 +25,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.RegionMovedException; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.util.Bytes; @@ -135,7 +135,7 @@ public void throwable(Throwable t, boolean retrying) { @Override public String getExceptionMessageAdditionalDetail() { - return "row '" + Bytes.toString(row) + "' on table '" + tableName + "' at " + location; + return "row '" + Bytes.toString(row) + "' on table '" + tableName; } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java index c6ed801138f5..cd494a46b913 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Registry.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.TableName; /** * Cluster registry. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java index 903b2f3305df..09c58cf2eebd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -29,14 +29,14 @@ import java.util.NavigableMap; import java.util.TreeMap; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java index 381505cba74a..8fd28b3454d0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultScanner.java @@ -18,12 +18,12 @@ */ package org.apache.hadoop.hbase.client; -import java.io.Closeable; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.Closeable; +import java.io.IOException; + /** * Interface for client-side scanning. * Go to {@link Table} to obtain instances. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java index 270671818745..e2663760c496 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java @@ -14,12 +14,13 @@ */ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + import java.io.IOException; import java.util.Date; import java.util.List; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.util.concurrent.Callable; /** * Exception thrown by HTable methods when an attempt to do something (like diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java index 650b5a3d4a13..253ff8b11330 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedWithDetailsException.java @@ -19,6 +19,11 @@ package org.apache.hadoop.hbase.client; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.util.Bytes; + import java.io.PrintWriter; import java.io.StringWriter; import java.util.Collection; @@ -28,11 +33,6 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.util.Bytes; - /** * This subclass of {@link org.apache.hadoop.hbase.client.RetriesExhaustedException} * is thrown when we have more information about which rows were causing which diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptor.java deleted file mode 100644 index f372e2d28658..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptor.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.io.IOException; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * This class is designed to fit into the RetryingCaller class which forms the - * central piece of intelligence for the client side retries for most calls. - * - * One can extend this class and intercept the RetryingCaller and add additional - * logic into the execution of a simple HTable operations like get, delete etc. - * - * Concrete implementations of this calls are supposed to the thread safe. The - * object is used across threads to identify the fast failing threads. - * - * For a concrete use case see {@link PreemptiveFastFailInterceptor} - * - * Example use case : - * try { - * interceptor.intercept - * doAction() - * } catch (Exception e) { - * interceptor.handleFailure - * } finally { - * interceptor.updateFaulireInfo - * } - * - * The {@link RetryingCallerInterceptor} also acts as a factory - * for getting a new {@link RetryingCallerInterceptorContext}. - * - */ - -@InterfaceAudience.Private -abstract class RetryingCallerInterceptor { - - protected RetryingCallerInterceptor() { - // Empty constructor protected for NoOpRetryableCallerInterceptor - } - - /** - * This returns the context object for the current call. - * - * @return context : the context that needs to be used during this call. - */ - public abstract RetryingCallerInterceptorContext createEmptyContext(); - - /** - * Call this function in case we caught a failure during retries. - * - * @param context - * : The context object that we obtained previously. - * @param t - * : The exception that we caught in this particular try - * @throws IOException - */ - public abstract void handleFailure(RetryingCallerInterceptorContext context, - Throwable t) throws IOException; - - /** - * Call this function alongside the actual call done on the callable. - * - * @param abstractRetryingCallerInterceptorContext - * @throws PreemptiveFastFailException - */ - public abstract void intercept( - RetryingCallerInterceptorContext abstractRetryingCallerInterceptorContext) - throws IOException; - - /** - * Call this function to update at the end of the retry. This is not necessary - * to happen. - * - * @param context - */ - public abstract void updateFailureInfo( - RetryingCallerInterceptorContext context); - - @Override - public abstract String toString(); -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java deleted file mode 100644 index a9f414f30d5c..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * The context object used in the {@link RpcRetryingCaller} to enable - * {@link RetryingCallerInterceptor} to intercept calls. - * {@link RetryingCallerInterceptorContext} is the piece of information unique - * to a retrying call that transfers information from the call into the - * {@link RetryingCallerInterceptor} so that {@link RetryingCallerInterceptor} - * can take appropriate action according to the specific logic - * - */ -@InterfaceAudience.Private -abstract class RetryingCallerInterceptorContext { - protected RetryingCallerInterceptorContext() { - } - - /** - * This function clears the internal state of the context object. - */ - public abstract void clear(); - - /** - * This prepares the context object by populating it with information specific - * to the implementation of the {@link RetryingCallerInterceptor} along with - * which this will be used. - * - * @param callable - * : The {@link RetryingCallable} that contains the information about - * the call that is being made. - * @return A new {@link RetryingCallerInterceptorContext} object that can be - * used for use in the current retrying call - */ - public abstract RetryingCallerInterceptorContext prepare( - RetryingCallable callable); - - /** - * Telescopic extension that takes which of the many retries we are currently - * in. - * - * @param callable - * : The {@link RetryingCallable} that contains the information about - * the call that is being made. - * @param tries - * : The retry number that we are currently in. - * @return A new context object that can be used for use in the current - * retrying call - */ - public abstract RetryingCallerInterceptorContext prepare( - RetryingCallable callable, int tries); -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorFactory.java deleted file mode 100644 index 9799ec006bd5..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorFactory.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import java.lang.reflect.Constructor; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.ConnectionManager.HConnectionImplementation; - -/** - * Factory implementation to provide the {@link HConnectionImplementation} with - * the implementation of the {@link RetryingCallerInterceptor} that we would use - * to intercept the {@link RpcRetryingCaller} during the course of their calls. - * - */ - -@InterfaceAudience.Private -class RetryingCallerInterceptorFactory { - private static final Log LOG = LogFactory - .getLog(RetryingCallerInterceptorFactory.class); - private Configuration conf; - private final boolean failFast; - public static final RetryingCallerInterceptor NO_OP_INTERCEPTOR = - new NoOpRetryableCallerInterceptor(null); - - public RetryingCallerInterceptorFactory(Configuration conf) { - this.conf = conf; - failFast = conf.getBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, - HConstants.HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT); - } - - /** - * This builds the implementation of {@link RetryingCallerInterceptor} that we - * specify in the conf and returns the same. - * - * To use {@link PreemptiveFastFailInterceptor}, set HBASE_CLIENT_ENABLE_FAST_FAIL_MODE to true. - * HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL is defaulted to {@link PreemptiveFastFailInterceptor} - * - * @return The factory build method which creates the - * {@link RetryingCallerInterceptor} object according to the - * configuration. - */ - public RetryingCallerInterceptor build() { - RetryingCallerInterceptor ret = NO_OP_INTERCEPTOR; - if (failFast) { - try { - Class c = conf.getClass( - HConstants.HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL, - PreemptiveFastFailInterceptor.class); - Constructor constructor = c - .getDeclaredConstructor(Configuration.class); - constructor.setAccessible(true); - ret = (RetryingCallerInterceptor) constructor.newInstance(conf); - } catch (Exception e) { - ret = new PreemptiveFastFailInterceptor(conf); - } - } - LOG.trace("Using " + ret.toString() + " for intercepting the RpcRetryingCaller"); - return ret; - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedClientScanner.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedClientScanner.java index b36cf7fa1daf..a03858e725be 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedClientScanner.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedClientScanner.java @@ -24,10 +24,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ExceptionUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedScannerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedScannerCallable.java index f8910b67a4dc..964dcb5bd53e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedScannerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ReversedScannerCallable.java @@ -23,13 +23,13 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowTooBigException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowTooBigException.java index 53ed7c9b47c0..3da3fd5e7a1c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowTooBigException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowTooBigException.java @@ -18,8 +18,8 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.RegionException; /** * Gets or Scans throw this exception if running without in-row scan flag diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java index 97e638120b18..a10f561eeaa1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCaller.java @@ -29,9 +29,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ExceptionUtil; import org.apache.hadoop.ipc.RemoteException; @@ -62,19 +61,10 @@ public class RpcRetryingCaller { private final long pause; private final int retries; private final AtomicBoolean cancelled = new AtomicBoolean(false); - private final RetryingCallerInterceptor interceptor; - private final RetryingCallerInterceptorContext context; public RpcRetryingCaller(long pause, int retries) { - this(pause, retries, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR); - } - - public RpcRetryingCaller(long pause, int retries, - RetryingCallerInterceptor interceptor) { this.pause = pause; this.retries = retries; - this.interceptor = interceptor; - context = interceptor.createEmptyContext(); } private int getRemainingTime(int callTimeout) { @@ -93,7 +83,7 @@ private int getRemainingTime(int callTimeout) { return remainingTime; } } - + public void cancel(){ cancelled.set(true); synchronized (cancelled){ @@ -114,15 +104,11 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) List exceptions = new ArrayList(); this.globalStartTime = EnvironmentEdgeManager.currentTime(); - context.clear(); for (int tries = 0;; tries++) { long expectedSleep; try { callable.prepare(tries != 0); // if called with false, check table status on ZK - interceptor.intercept(context.prepare(callable, tries)); return callable.call(getRemainingTime(callTimeout)); - } catch (PreemptiveFastFailException e) { - throw e; } catch (Throwable t) { ExceptionUtil.rethrowIfInterrupt(t); if (LOG.isTraceEnabled()) { @@ -132,7 +118,6 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) } // translateException throws exception when should not retry: i.e. when request is bad. - interceptor.handleFailure(context, t); t = translateException(t); callable.throwable(t, retries != 1); RetriesExhaustedException.ThrowableWithExtraContext qt = @@ -154,8 +139,6 @@ public T callWithRetries(RetryingCallable callable, int callTimeout) ": " + callable.getExceptionMessageAdditionalDetail(); throw (SocketTimeoutException)(new SocketTimeoutException(msg).initCause(t)); } - } finally { - interceptor.updateFailureInfo(context); } try { if (expectedSleep > 0) { @@ -205,7 +188,7 @@ public T callWithoutRetries(RetryingCallable callable, int callTimeout) } } } - + /** * Get the good or the remote exception if any, throws the DoNotRetryIOException. * @param t the throwable to analyze diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java index f482262ad2fc..9b070a57b334 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerFactory.java @@ -31,38 +31,27 @@ public class RpcRetryingCallerFactory { protected final Configuration conf; private final long pause; private final int retries; - private final RetryingCallerInterceptor interceptor; public RpcRetryingCallerFactory(Configuration conf) { - this(conf, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR); - } - - public RpcRetryingCallerFactory(Configuration conf, RetryingCallerInterceptor interceptor) { this.conf = conf; pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE); retries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - this.interceptor = interceptor; } public RpcRetryingCaller newCaller() { // We store the values in the factory instance. This way, constructing new objects // is cheap as it does not require parsing a complex structure. - return new RpcRetryingCaller(pause, retries, interceptor); + return new RpcRetryingCaller(pause, retries); } public static RpcRetryingCallerFactory instantiate(Configuration configuration) { - return instantiate(configuration, RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR); - } - - public static RpcRetryingCallerFactory instantiate(Configuration configuration, - RetryingCallerInterceptor interceptor) { String clazzName = RpcRetryingCallerFactory.class.getName(); String rpcCallerFactoryClazz = configuration.get(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, clazzName); if (rpcCallerFactoryClazz.equals(clazzName)) { - return new RpcRetryingCallerFactory(configuration, interceptor); + return new RpcRetryingCallerFactory(configuration); } return ReflectionUtils.instantiateWithCustomCtor(rpcCallerFactoryClazz, new Class[] { Configuration.class }, new Object[] { configuration }); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java index 85704ffb5d7a..4c5814e61158 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RpcRetryingCallerWithReadReplicas.java @@ -21,19 +21,7 @@ package org.apache.hadoop.hbase.client; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.RunnableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - +import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -50,8 +38,18 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import com.google.protobuf.ServiceException; -import org.htrace.Trace; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * Caller that goes to replica if the primary region does no answer within a configurable @@ -414,7 +412,7 @@ public ResultBoundedCompletionService(Executor executor, int maxTasks) { public void submit(ReplicaRegionServerCallable task, int callTimeout) { QueueingFuture newFuture = new QueueingFuture(task, callTimeout); - executor.execute(Trace.wrap(newFuture)); + executor.execute(newFuture); tasks[task.id] = newFuture; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java index 2aea19f31fab..ae56dc9dac1b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -31,9 +31,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.IncompatibleFilterException; import org.apache.hadoop.hbase.io.TimeRange; @@ -213,7 +213,6 @@ public Scan(Scan scan) throws IOException { loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue(); consistency = scan.getConsistency(); reversed = scan.isReversed(); - small = scan.isSmall(); TimeRange ctr = scan.getTimeRange(); tr = new TimeRange(ctr.getMin(), ctr.getMax()); Map> fams = scan.getFamilyMap(); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 48dea4e6de1d..e24af8948db9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; @@ -37,7 +38,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownScannerException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; @@ -293,7 +293,7 @@ private void updateResultsMetrics(Result[] rrs) { long resultSize = 0; for (Result rr : rrs) { for (Cell cell : rr.rawCells()) { - resultSize += CellUtil.estimatedSerializedSizeOf(cell); + resultSize += CellUtil.estimatedLengthOf(cell); } } this.scanMetrics.countOfBytesInResults.addAndGet(resultSize); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallableWithReplicas.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallableWithReplicas.java index 0de658bfd2e1..440cddf18eb6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallableWithReplicas.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallableWithReplicas.java @@ -34,11 +34,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.BoundedCompletionService; import org.apache.hadoop.hbase.util.Pair; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java index 9e0827c9cfff..63ec370fff32 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerTimeoutException.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Thrown when a scanner has timed out. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java index a408b1d35df7..07e4c080f32e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java @@ -23,20 +23,20 @@ import java.util.List; import java.util.Map; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import com.google.protobuf.Service; +import com.google.protobuf.ServiceException; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; -import com.google.protobuf.Descriptors; -import com.google.protobuf.Message; -import com.google.protobuf.Service; -import com.google.protobuf.ServiceException; - /** * Used to communicate with a single HBase table. * Obtain an instance from a {@link Connection} and call {@link #close()} afterwards. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableState.java index be9b80ce1695..930098e18235 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableState.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java index 33aef792ffaa..7fe2a7300d47 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHRegionInfo.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HRegionInfo; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java index 322a55bf3e2b..9a58837fc0e3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/UnmodifyableHTableDescriptor.java @@ -19,10 +19,10 @@ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; /** * Read-only table descriptor. @@ -67,11 +67,6 @@ public HTableDescriptor addFamily(final HColumnDescriptor family) { throw new UnsupportedOperationException("HTableDescriptor is read-only"); } - @Override - public HTableDescriptor modifyFamily(HColumnDescriptor family) { - throw new UnsupportedOperationException("HTableDescriptor is read-only"); - } - /** * @param column * @return Column descriptor for the passed family name or the family on diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/WrongRowIOException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/WrongRowIOException.java index e0609dab9ee3..09c1d6438779 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/WrongRowIOException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/WrongRowIOException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperKeepAliveConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperKeepAliveConnection.java index 04fd20fcc929..9b987b5eebb5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperKeepAliveConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperKeepAliveConnection.java @@ -20,11 +20,11 @@ package org.apache.hadoop.hbase.client; -import java.io.IOException; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import java.io.IOException; + /** * We inherit the current ZooKeeperWatcher implementation to change the semantic * of the close: the new close won't immediately close the connection but diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java index 11a095e542dc..7cab2cc85088 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java @@ -23,8 +23,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java index 85ce4e288bc4..ea0e1dfd084a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/AggregationClient.java @@ -31,11 +31,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java index e14bda669cdc..8fc00a3feea1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/Batch.java @@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.client.coprocessor; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * A collection of interfaces and utilities used for interacting with custom RPC * interfaces exposed by Coprocessors. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java index 97724bdaa241..d693f0c20cef 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java @@ -22,13 +22,13 @@ import java.math.BigDecimal; import java.math.RoundingMode; +import org.apache.hadoop.hbase.util.ByteStringer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BigDecimalMsg; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.EmptyMsg; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java index 8b0c690831cd..9aac6bf87da7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java @@ -20,12 +20,13 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.DoubleMsg; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.EmptyMsg; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.DoubleMsg; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java index e8e5e3a7e114..e63fd3ba36bf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java @@ -20,8 +20,8 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.EmptyMsg; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.LongMsg; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java index 0fcd4193e4c0..90564e47dd50 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/SecureBulkLoadClient.java @@ -21,14 +21,12 @@ import static org.apache.hadoop.hbase.HConstants.EMPTY_START_ROW; import static org.apache.hadoop.hbase.HConstants.LAST_ROW; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.ByteStringer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.ipc.ServerRpcController; @@ -36,10 +34,13 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos; import org.apache.hadoop.hbase.security.SecureBulkLoadUtil; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.security.token.Token; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + /** * Client proxy for SecureBulkLoadProtocol * used in conjunction with SecureBulkLoadEndpoint diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java index 3a337602a121..cf3ad7a30146 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,14 +33,14 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.replication.ReplicationException; @@ -48,7 +49,6 @@ import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.ReplicationQueuesClient; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java index 708f1899e6e5..00ae239351b3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java @@ -21,8 +21,10 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter; import com.google.protobuf.Message; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java index 9946d9739c29..330b9d5e7323 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.coprocessor; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Thrown if a coprocessor encounters any exception. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ConnectionClosingException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ConnectionClosingException.java deleted file mode 100644 index cb8e5df4c9a5..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/ConnectionClosingException.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.exceptions; - -/** -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -import java.io.IOException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** -* Thrown when the client believes that we are trying to communicate to has -* been repeatedly unresponsive for a while. -* -* On receiving such an exception. The HConnectionManager will skip all -* retries and fast fail the operation. -*/ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public class ConnectionClosingException extends IOException { - public ConnectionClosingException(String string) { - super(string); - } - - private static final long serialVersionUID = -8980028569652624236L; -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/MergeRegionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/MergeRegionException.java index 3042ad81d31b..cb34d3d9e335 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/MergeRegionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/MergeRegionException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.exceptions; -import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.RegionException; /** * Thrown when something is wrong in trying to merge two regions. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OperationConflictException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OperationConflictException.java index c40b8d9dad00..ca5eeb034187 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OperationConflictException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OperationConflictException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.exceptions; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * The exception that is thrown if there's duplicate execution of non-idempotent operation. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OutOfOrderScannerNextException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OutOfOrderScannerNextException.java index 6357bd6594df..5a49be1884ed 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OutOfOrderScannerNextException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/OutOfOrderScannerNextException.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.exceptions; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Thrown by a RegionServer while doing next() calls on a ResultScanner. Both client and server diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/PreemptiveFastFailException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/PreemptiveFastFailException.java deleted file mode 100644 index 2d66d54852bf..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/PreemptiveFastFailException.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.hadoop.hbase.exceptions; - -import java.net.ConnectException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hbase.ServerName; - -/** - * Thrown when the client believes that we are trying to communicate to has - * been repeatedly unresponsive for a while. - * - * On receiving such an exception. The HConnectionManager will skip all - * retries and fast fail the operation. - */ - @InterfaceAudience.Public - @InterfaceStability.Evolving - public class PreemptiveFastFailException extends ConnectException { - private static final long serialVersionUID = 7129103682617007177L; - private long failureCount, timeOfFirstFailureMilliSec, timeOfLatestAttemptMilliSec; - - /** - * @param count - * @param timeOfFirstFailureMilliSec - * @param timeOfLatestAttemptMilliSec - * @param serverName - */ - public PreemptiveFastFailException(long count, long timeOfFirstFailureMilliSec, - long timeOfLatestAttemptMilliSec, ServerName serverName) { - super("Exception happened " + count + " times. to" + serverName); - this.failureCount = count; - this.timeOfFirstFailureMilliSec = timeOfFirstFailureMilliSec; - this.timeOfLatestAttemptMilliSec = timeOfLatestAttemptMilliSec; - } - - public long getFirstFailureAt() { - return timeOfFirstFailureMilliSec; - } - - public long getLastAttemptAt() { - return timeOfLatestAttemptMilliSec; - } - - public long getFailureCount() { - return failureCount; - } - - public boolean wasOperationAttemptedByServer() { - return false; - } - } \ No newline at end of file diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionInRecoveryException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionInRecoveryException.java index 06db472f7b3f..78b7c427abec 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionInRecoveryException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionInRecoveryException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.exceptions; -import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.NotServingRegionException; /** * Thrown when a read request issued against a region which is in recovering state. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionMovedException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionMovedException.java index c76be0faa950..c7bd3f00d0f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionMovedException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionMovedException.java @@ -19,11 +19,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; /** * Subclass if the server knows the region is now on another server. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionOpeningException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionOpeningException.java index 8833372f3e83..efb79f9e1ed7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionOpeningException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/exceptions/RegionOpeningException.java @@ -20,9 +20,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.ipc.RemoteException; /** * Subclass if the server knows the region is now on another server. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java index 0fc624ffc8a7..d5c2613f5fb5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryComparator.java @@ -19,14 +19,13 @@ package org.apache.hadoop.hbase.filter; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.InvalidProtocolBufferException; - /** * A binary comparator which lexicographically compares against the specified * byte array using {@link org.apache.hadoop.hbase.util.Bytes#compareTo(byte[], byte[])}. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java index b6311b022f00..c05eb8f42f48 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BinaryPrefixComparator.java @@ -19,14 +19,13 @@ package org.apache.hadoop.hbase.filter; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.InvalidProtocolBufferException; - /** * A comparator which compares against a specified byte array, but only compares * up to the length of this byte array. For the rest it is similar to diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java index d527a2cc2d5b..0b7c52db9b5b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BitComparator.java @@ -19,13 +19,12 @@ package org.apache.hadoop.hbase.filter; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; -import com.google.protobuf.InvalidProtocolBufferException; - /** * A bit comparator which performs the specified bitwise operation on each of the bytes * with the specified byte array. Then returns whether the result is non-zero. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java index 4642ab9775a8..c6e42d46f7d7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase.filter; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java index 572de9f5c7a0..6107f32614f0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnCountGetFilter.java @@ -21,9 +21,9 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java index 673ca6e37c1b..97956954f588 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java @@ -20,10 +20,10 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java index d2f058a59cd7..c8474baf5d53 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnPrefixFilter.java @@ -21,10 +21,10 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java index 9963af6451fb..e75d8bac6135 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ColumnRangeFilter.java @@ -23,10 +23,10 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java index 319e1238b553..567f8413c3af 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/CompareFilter.java @@ -19,8 +19,7 @@ package org.apache.hadoop.hbase.filter; -import java.util.ArrayList; - +import com.google.common.base.Preconditions; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -29,7 +28,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.CompareType; import org.apache.hadoop.hbase.util.Bytes; -import com.google.common.base.Preconditions; +import java.util.ArrayList; /** * This is a generic filter to be used to filter by comparison. It takes an * operator (equal, greater, not equal, etc) and a byte [] comparator. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java index 4fe052266a1e..5cfedaaeb128 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/DependentColumnFilter.java @@ -25,14 +25,14 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.base.Preconditions; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java index e2890268e05e..c7b0b665e28e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FamilyFilter.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java index d66ad50d8297..d5d87c3e0a0d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/Filter.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.util.List; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java index a04dd897f428..2255097e6c0a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java @@ -23,8 +23,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; /** * Abstract base class to help you implement new Filters. Common "ignore" or NOOP type diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java index ba1a8188ad69..d98ca8bf15fa 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java @@ -23,11 +23,11 @@ import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java index 517611525a77..a5a0eeee1858 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.util.List; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java index 77ed7d996973..ffcf78a3cbc4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.java @@ -20,9 +20,9 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyValueMatchingQualifiersFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyValueMatchingQualifiersFilter.java index 622f5ab5aefe..fc40982a1d15 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyValueMatchingQualifiersFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FirstKeyValueMatchingQualifiersFilter.java @@ -18,13 +18,10 @@ package org.apache.hadoop.hbase.filter; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; @@ -33,6 +30,9 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Set; +import java.util.TreeSet; + /** * The filter looks for the given columns in KeyValue. Once there is a match for * any one of the columns, it returns ReturnCode.NEXT_ROW for remaining diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java index 9b99b710d733..39713307a15b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -21,10 +21,11 @@ import java.util.Arrays; import java.util.List; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValueUtil; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; @@ -32,7 +33,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.InvalidProtocolBufferException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java index cf2d1533ce84..4ca6cf81e708 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java @@ -21,12 +21,12 @@ import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.base.Preconditions; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java index 2a2b525efe51..cebb26ac880d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.util.ArrayList; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/LongComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/LongComparator.java deleted file mode 100644 index 38854d4755f3..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/LongComparator.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.filter; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * A long comparator which numerical compares against the specified byte array - */ - -public class LongComparator extends ByteArrayComparable { - private Long longValue; - - public LongComparator(long value) { - super(Bytes.toBytes(value)); - this.longValue = value; - } - - @Override - public int compareTo(byte[] value, int offset, int length) { - Long that = Bytes.toLong(value, offset, length); - return this.longValue.compareTo(that); - } - - /** - * @return The comparator serialized using pb - */ - public byte [] toByteArray() { - ComparatorProtos.LongComparator.Builder builder = - ComparatorProtos.LongComparator.newBuilder(); - builder.setComparable(super.convert()); - return builder.build().toByteArray(); - } - - /** - * @param pbBytes A pb serialized {@link BinaryComparator} instance - * @return An instance of {@link BinaryComparator} made from bytes - * @throws org.apache.hadoop.hbase.exceptions.DeserializationException - * @see #toByteArray - */ - public static LongComparator parseFrom(final byte [] pbBytes) - throws DeserializationException { - ComparatorProtos.LongComparator proto; - try { - proto = ComparatorProtos.LongComparator.parseFrom(pbBytes); - } catch (InvalidProtocolBufferException e) { - throw new DeserializationException(e); - } - return new LongComparator(Bytes.toLong(proto.getComparable().getValue().toByteArray())); - } - - /** - * @param other - * @return true if and only if the fields of the comparator that are serialized - * are equal to the corresponding fields in other. Used for testing. - */ - boolean areSerializedFieldsEqual(LongComparator other) { - if (other == this) return true; - if (!(other instanceof LongComparator)) return false; - - return super.areSerializedFieldsEqual(other); - } -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java index b7ec11a20e1c..700d6db5b1d7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/MultipleColumnPrefixFilter.java @@ -22,10 +22,10 @@ import java.util.Comparator; import java.util.TreeSet; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java index 0dbd97b3b06a..060219e2dcc7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PageFilter.java @@ -18,17 +18,17 @@ */ package org.apache.hadoop.hbase.filter; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.hadoop.hbase.Cell; +import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.util.ArrayList; /** * Implementation of Filter interface that limits results to a specific page * size. It terminates scanning once the number of filter-passed rows is > diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java index 3a2077237274..8e6cfae093cb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseConstants.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase.filter; -import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.nio.ByteBuffer; + /** * ParseConstants holds a bunch of constants related to parsing Filter Strings * Used by {@link ParseFilter} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java index 8101f4aa4f08..9d6bd3cb0129 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java @@ -18,6 +18,14 @@ */ package org.apache.hadoop.hbase.filter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.util.Bytes; + import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; @@ -30,14 +38,6 @@ import java.util.Set; import java.util.Stack; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; -import org.apache.hadoop.hbase.util.Bytes; - /** * This class allows a user to specify a filter via a string * The string is parsed using the methods of this class and diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java index 5b56748c0688..323cd691de67 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/PrefixFilter.java @@ -19,18 +19,18 @@ package org.apache.hadoop.hbase.filter; -import java.util.ArrayList; +import com.google.common.base.Preconditions; +import org.apache.hadoop.hbase.util.ByteStringer; +import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; +import java.util.ArrayList; /** * Pass results that have same row prefix. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java index e1c4c57fb030..ff2a0da04cc3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/QualifierFilter.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java index 2a25b32b58f0..bff404cede60 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RandomRowFilter.java @@ -21,9 +21,9 @@ import java.util.Random; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java index 127aa4c3e5ea..6e4f7d028503 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java @@ -18,6 +18,8 @@ */ package org.apache.hadoop.hbase.filter; +import com.google.protobuf.InvalidProtocolBufferException; + import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.util.Arrays; @@ -30,6 +32,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; import org.apache.hadoop.hbase.util.Bytes; + import org.jcodings.Encoding; import org.jcodings.EncodingDB; import org.jcodings.specific.UTF8Encoding; @@ -38,8 +41,6 @@ import org.joni.Regex; import org.joni.Syntax; -import com.google.protobuf.InvalidProtocolBufferException; - /** * This comparator is for use with {@link CompareFilter} implementations, such * as {@link RowFilter}, {@link QualifierFilter}, and {@link ValueFilter}, for diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java index b6407f1f9372..d975486fa938 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/RowFilter.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.util.ArrayList; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java index d030fd223241..af0ec1d4bd0b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueExcludeFilter.java @@ -24,10 +24,10 @@ import java.util.Iterator; import java.util.List; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java index e8f8f98535ea..f72d653b42f5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java @@ -22,12 +22,13 @@ import java.io.IOException; import java.util.ArrayList; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -35,7 +36,6 @@ import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.CompareType; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.base.Preconditions; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java index ce8e5112b397..f6d120b62594 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java @@ -21,9 +21,9 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java index 63fd0a38cd27..5eb3703603bb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SubstringComparator.java @@ -18,14 +18,13 @@ */ package org.apache.hadoop.hbase.filter; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos; import org.apache.hadoop.hbase.util.Bytes; -import com.google.protobuf.InvalidProtocolBufferException; - /** * This comparator is for use with SingleColumnValueFilter, for filtering based on diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java index 32a3d7311776..5c535242ad73 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/TimestampsFilter.java @@ -17,18 +17,17 @@ */ package org.apache.hadoop.hbase.filter; -import java.util.ArrayList; -import java.util.List; -import java.util.TreeSet; - -import org.apache.hadoop.hbase.Cell; +import com.google.common.base.Preconditions; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; /** * Filter that returns only cells whose timestamp (version) is diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java index a2c5eb29ebde..5a46d7a11e48 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ValueFilter.java @@ -19,17 +19,16 @@ package org.apache.hadoop.hbase.filter; -import java.io.IOException; -import java.util.ArrayList; - -import org.apache.hadoop.hbase.Cell; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; -import com.google.protobuf.InvalidProtocolBufferException; +import java.io.IOException; +import java.util.ArrayList; /** * This filter is used to filter based on column value. It takes an diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java index 31d4f77e3239..7263e1bfcd63 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java @@ -21,9 +21,9 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java index 938797c6565d..c10a098e2c35 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java @@ -18,10 +18,6 @@ package org.apache.hadoop.hbase.ipc; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.protobuf.ResponseConverter; @@ -34,6 +30,10 @@ import com.google.protobuf.RpcController; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.IOException; /** * Base class which provides clients with an RPC connection to diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FatalConnectionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FatalConnectionException.java index 86d3b89e01b1..9bfb651e493b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FatalConnectionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FatalConnectionException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Thrown when server finds fatal issue w/ connection setup: e.g. bad rpc version diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java index 1bef719e34db..118a8952d858 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java @@ -20,15 +20,15 @@ import java.io.IOException; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse; -import org.apache.hadoop.hbase.util.ByteStringer; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java index 141f2344eb41..3e7a7a2c7f99 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java @@ -19,12 +19,12 @@ import java.util.List; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * Optionally carries Cells across the proxy/service interface down into ipc. On its diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java index 8c75f4fa3f0d..026c64bdc883 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java @@ -20,18 +20,18 @@ import java.io.IOException; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.RegionServerCallable; import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import com.google.protobuf.Descriptors; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java index 122bfdc03c49..cf22dd59c2cc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse; import org.apache.hadoop.hbase.util.ByteStringer; + import com.google.protobuf.Descriptors; import com.google.protobuf.Message; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RemoteWithExtrasException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RemoteWithExtrasException.java index a80cbf355782..47e89371ade0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RemoteWithExtrasException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RemoteWithExtrasException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.ipc.RemoteException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java index 4586e3ebb40b..522971bba2fc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java @@ -19,51 +19,26 @@ package org.apache.hadoop.hbase.ipc; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.Closeable; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InterruptedIOException; -import java.io.OutputStream; -import java.net.ConnectException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketAddress; -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.security.PrivilegedExceptionAction; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.net.SocketFactory; -import javax.security.sasl.SaslException; - +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.BlockingRpcChannel; +import com.google.protobuf.Descriptors.MethodDescriptor; +import com.google.protobuf.Message; +import com.google.protobuf.Message.Builder; +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.codec.KeyValueCodec; -import org.apache.hadoop.hbase.exceptions.ConnectionClosingException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta; @@ -99,14 +74,37 @@ import org.htrace.Trace; import org.htrace.TraceScope; -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.BlockingRpcChannel; -import com.google.protobuf.Descriptors.MethodDescriptor; -import com.google.protobuf.Message; -import com.google.protobuf.Message.Builder; -import com.google.protobuf.RpcCallback; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; +import javax.net.SocketFactory; +import javax.security.sasl.SaslException; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.net.ConnectException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + /** * Does RPC against a cluster. Manages connections per regionserver in the cluster. @@ -338,8 +336,7 @@ public long getStartTime() { protected final static Map> tokenHandlers = - new HashMap>(); + new HashMap>(); static { tokenHandlers.put(AuthenticationProtos.TokenIdentifier.Kind.HBASE_AUTH_TOKEN, new AuthenticationTokenSelector()); @@ -503,7 +500,7 @@ public void run() { private void cleanup() { assert shouldCloseConnection.get(); - IOException ie = new ConnectionClosingException("Connection to " + server + " is closing."); + IOException ie = new IOException("Connection to " + server + " is closing."); while (true) { CallFuture cts = callsToWrite.poll(); if (cts == null) { @@ -653,21 +650,18 @@ protected synchronized void closeConnection() { socket.getOutputStream().close(); } } catch (IOException ignored) { // Can happen if the socket is already closed - if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored); } try { if (socket.getInputStream() != null) { socket.getInputStream().close(); } } catch (IOException ignored) { // Can happen if the socket is already closed - if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored); } try { if (socket.getChannel() != null) { socket.getChannel().close(); } } catch (IOException ignored) { // Can happen if the socket is already closed - if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored); } try { socket.close(); @@ -721,7 +715,7 @@ private void handleConnectionFailure(int curRetries, int maxRetries, IOException */ private void checkIsOpen() throws IOException { if (shouldCloseConnection.get()) { - throw new ConnectionClosingException(getName() + " is closing"); + throw new IOException(getName() + " is closing"); } } @@ -911,7 +905,7 @@ protected synchronized void setupIOstreams() throws IOException { } if (shouldCloseConnection.get()){ - throw new ConnectionClosingException("This connection is closing"); + throw new IOException("This connection is closing"); } if (failedServers.isFailedServer(remoteId.getAddress())) { @@ -1165,18 +1159,18 @@ protected void readResponse() { int readSoFar = IPCUtil.getTotalSizeWhenWrittenDelimited(responseHeader); int whatIsLeftToRead = totalSize - readSoFar; IOUtils.skipFully(in, whatIsLeftToRead); - return; } if (responseHeader.hasException()) { ExceptionResponse exceptionResponse = responseHeader.getException(); RemoteException re = createRemoteException(exceptionResponse); - call.setException(re); + if (expectedCall) call.setException(re); if (isFatalConnectionException(exceptionResponse)) { markClosed(re); } } else { Message value = null; - if (call.responseDefaultType != null) { + // Call may be null because it may have timeout and been cleaned up on this side already + if (expectedCall && call.responseDefaultType != null) { Builder builder = call.responseDefaultType.newBuilderForType(); builder.mergeDelimitedFrom(in); value = builder.build(); @@ -1188,7 +1182,9 @@ protected void readResponse() { IOUtils.readFully(this.in, cellBlock, 0, cellBlock.length); cellBlockScanner = ipcUtil.createCellScanner(this.codec, this.compressor, cellBlock); } - call.setResponse(value, cellBlockScanner); + // it's possible that this call may have been cleaned up due to a RPC + // timeout, so check if it still exists before setting the value. + if (expectedCall) call.setResponse(value, cellBlockScanner); } } catch (IOException e) { if (expectedCall) call.setException(e); @@ -1196,7 +1192,6 @@ protected void readResponse() { // Clean up open calls but don't treat this as a fatal condition, // since we expect certain responses to not make it by the specified // {@link ConnectionId#rpcTimeout}. - if (LOG.isTraceEnabled()) LOG.trace("ignored", e); } else { // Treat this as a fatal condition and close this connection markClosed(e); @@ -1257,7 +1252,7 @@ protected synchronized void cleanupCalls(boolean allCalls) { itor.remove(); } else if (allCalls) { long waitTime = EnvironmentEdgeManager.currentTime() - c.getStartTime(); - IOException ie = new ConnectionClosingException("Connection to " + getRemoteAddress() + IOException ie = new IOException("Connection to " + getRemoteAddress() + " is closing. Call id=" + c.id + ", waitTime=" + waitTime); c.setException(ie); itor.remove(); @@ -1492,6 +1487,7 @@ Pair call(PayloadCarryingRpcController pcrc, MethodDescrip @Override public void run(Object parameter) { connection.callSender.remove(cts); + call.callComplete(); } }); if (pcrc.isCanceled()) { @@ -1512,8 +1508,8 @@ public void run(Object parameter) { break; } if (connection.shouldCloseConnection.get()) { - throw new ConnectionClosingException("Call id=" + call.id + - " on server " + addr + " aborted: connection is closing"); + throw new IOException("Call id=" + call.id + " on server " + + addr + " aborted: connection is closing"); } try { synchronized (call) { @@ -1561,9 +1557,6 @@ protected IOException wrapException(InetSocketAddress addr, } else if (exception instanceof SocketTimeoutException) { return (SocketTimeoutException)new SocketTimeoutException("Call to " + addr + " failed because " + exception).initCause(exception); - } else if (exception instanceof ConnectionClosingException){ - return (ConnectionClosingException) new ConnectionClosingException( - "Call to " + addr + " failed on local exception: " + exception).initCause(exception); } else { return (IOException)new IOException("Call to " + addr + " failed on local exception: " + exception).initCause(exception); @@ -1664,7 +1657,7 @@ public boolean equals(Object obj) { public int hashCode() { int hashcode = (address.hashCode() + PRIME * (PRIME * this.serviceName.hashCode() ^ - (ticket == null ? 0 : ticket.hashCode()))); + (ticket == null ? 0 : ticket.hashCode()) )); return hashcode; } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java index 6d0b9deccc0c..07d8683c0886 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.java @@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.ipc; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + @SuppressWarnings("serial") @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java index 0a4354adf080..cbf63fc37c10 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java @@ -18,15 +18,14 @@ package org.apache.hadoop.hbase.ipc; -import java.io.IOException; - -import org.apache.hadoop.util.StringUtils; - import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; import com.google.protobuf.Service; +import org.apache.hadoop.util.StringUtils; + +import java.io.IOException; /** * Used for server-side protobuf RPC service invocations. This handler allows diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/StoppedRpcClientException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/StoppedRpcClientException.java index a224a1232037..c11cb2878d34 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/StoppedRpcClientException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/StoppedRpcClientException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HBaseIOException; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java index a56f904c8063..ec98a5f26b61 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java @@ -19,11 +19,12 @@ package org.apache.hadoop.hbase.ipc; -import java.util.concurrent.atomic.AtomicReference; - import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + public class TimeLimitedRpcController implements RpcController { /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index 34c43ce05f63..dbcbaeb0e984 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -19,10 +19,10 @@ import java.util.Date; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index 768da760c880..fd9d1abf561f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -1447,9 +1447,7 @@ public static Filter toFilter(FilterProtos.Filter proto) throws IOException { } return (Filter)parseFrom.invoke(c, value); } catch (Exception e) { - // Either we couldn't instantiate the method object, or "parseFrom" failed. - // In either case, let's not retry. - throw new DoNotRetryIOException(e); + throw new IOException(e); } } @@ -2281,7 +2279,7 @@ public static List getUserPermissions( AccessControlProtos.GetUserPermissionsRequest request = builder.build(); AccessControlProtos.GetUserPermissionsResponse response = protocol.getUserPermissions(null, request); - List perms = new ArrayList(response.getUserPermissionCount()); + List perms = new ArrayList(); for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) { perms.add(ProtobufUtil.toUserPermission(perm)); } @@ -2309,7 +2307,7 @@ public static List getUserPermissions( AccessControlProtos.GetUserPermissionsRequest request = builder.build(); AccessControlProtos.GetUserPermissionsResponse response = protocol.getUserPermissions(null, request); - List perms = new ArrayList(response.getUserPermissionCount()); + List perms = new ArrayList(); for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) { perms.add(ProtobufUtil.toUserPermission(perm)); } @@ -2337,7 +2335,7 @@ public static List getUserPermissions( AccessControlProtos.GetUserPermissionsRequest request = builder.build(); AccessControlProtos.GetUserPermissionsResponse response = protocol.getUserPermissions(null, request); - List perms = new ArrayList(response.getUserPermissionCount()); + List perms = new ArrayList(); for (AccessControlProtos.UserPermission perm: response.getUserPermissionList()) { perms.add(ProtobufUtil.toUserPermission(perm)); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java index 6be8d2d8c49d..a0aaac1b22c0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java @@ -20,6 +20,9 @@ import java.io.IOException; import java.util.List; +import org.apache.hadoop.hbase.util.ByteStringer; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -28,7 +31,6 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Action; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; @@ -88,7 +90,6 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableStateRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest; @@ -106,6 +107,8 @@ import com.google.protobuf.ByteString; +import static org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*; + /** * Helper utility to build protocol buffer requests, * or build components for protocol buffer requests. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java index ced458a504b5..ca6b67eb2633 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java @@ -23,12 +23,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse; @@ -40,11 +40,11 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse; @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.StringUtils; import com.google.protobuf.ByteString; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java index f13ce2824250..2319b7aebac3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java @@ -26,14 +26,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.util.StringUtils; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java index 592c4dba86aa..6654f9e54084 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java @@ -19,11 +19,12 @@ import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; +import org.apache.hadoop.hbase.quotas.ThrottleType; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java index e29fef16e3d0..d15029e7cd62 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java @@ -21,13 +21,13 @@ import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 61538764b1f5..b758df67304e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -21,27 +21,28 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; + import java.util.List; import java.util.Map; import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.QualifierFilter; -import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java index 1f98eb9c7f95..fab2bd04d7d9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java @@ -17,14 +17,17 @@ */ package org.apache.hadoop.hbase.quotas; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottlingException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottlingException.java index bcad9431f757..bb24f86a9b37 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottlingException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottlingException.java @@ -21,6 +21,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hadoop.util.StringUtils; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java index d1fdae34310c..de1726b97184 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/LeaseException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Reports a problem with a lease diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java index d3b1ec1d9935..4b541c8d465a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/NoSuchColumnFamilyException.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.regionserver; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * Thrown if request for nonexistent column family. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java index c2460d4f287c..4338539b27d3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/WrongRegionException.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase.regionserver; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown when a request contains a key which is not part of this region */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java index cc42819bdd09..8af8d75ddbe8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FailedLogCloseException.java @@ -18,11 +18,11 @@ */ package org.apache.hadoop.hbase.regionserver.wal; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * Thrown when we fail close of the write-ahead-log file. * Package private. Only used inside this package. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java index b8b5b224ad55..6925778f4303 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java @@ -20,10 +20,10 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * ReplicationPeer manages enabled / disabled state for the peer. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java index 6d42d5bc7452..848303cbb5f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java @@ -20,16 +20,18 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java index bd4eb30c3d1d..a601fb52072f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java @@ -18,14 +18,13 @@ */ package org.apache.hadoop.hbase.replication; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Pair; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java index 91e0b05c929a..cc49a640a070 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java @@ -42,8 +42,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; import org.apache.zookeeper.KeeperException; import com.google.protobuf.ByteString; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java index cd19157faebd..0664923a01cd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java @@ -19,14 +19,14 @@ package org.apache.hadoop.hbase.replication; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.ServerName; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + /** * This class is responsible for the parsing logic for a znode representing a queue. * It will extract the peerId if it's recovered as well as the dead region servers diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java index 282854673879..0ad94b3c027c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java @@ -35,8 +35,8 @@ import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp; import org.apache.zookeeper.KeeperException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java index 51d7473807e5..b469df48776f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java @@ -18,10 +18,10 @@ */ package org.apache.hadoop.hbase.replication; -import java.util.List; - import org.apache.hadoop.hbase.classification.InterfaceAudience; +import java.util.List; + /** * This is the interface for a Replication Tracker. A replication tracker provides the facility to * subscribe and track events that reflect a change in replication state. These events are used by diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java index 07b871d55128..8b446f5e1f0e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AccessDeniedException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.security; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AuthMethod.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AuthMethod.java index e2cc3cf8f358..38065fd1f61c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AuthMethod.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/AuthMethod.java @@ -19,13 +19,13 @@ package org.apache.hadoop.hbase.security; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.security.UserGroupInformation; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.security.UserGroupInformation; - /** Authentication method */ @InterfaceAudience.Private public enum AuthMethod { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java index f4bc3e9c6811..f446c6649e35 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/EncryptionUtil.java @@ -26,14 +26,14 @@ import javax.crypto.spec.SecretKeySpec; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.protobuf.generated.EncryptionProtos; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java index 895d06712630..74a9f35c9736 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/HBaseSaslRpcClient.java @@ -18,13 +18,15 @@ package org.apache.hadoop.hbase.security; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.SaslInputStream; +import org.apache.hadoop.security.SaslOutputStream; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; @@ -37,15 +39,13 @@ import javax.security.sasl.SaslClient; import javax.security.sasl.SaslException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.security.SaslInputStream; -import org.apache.hadoop.security.SaslOutputStream; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import com.google.common.annotations.VisibleForTesting; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SaslUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SaslUtil.java index 726a75364c03..9cde790b35cf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SaslUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SaslUtil.java @@ -18,14 +18,14 @@ */ package org.apache.hadoop.hbase.security; +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + import java.util.Map; import java.util.TreeMap; import javax.security.sasl.Sasl; -import org.apache.commons.codec.binary.Base64; -import org.apache.hadoop.hbase.classification.InterfaceAudience; - @InterfaceAudience.Private public class SaslUtil { public static final String SASL_DEFAULT_REALM = "default"; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java index 30959a0a999f..2fde925511e4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java @@ -18,9 +18,9 @@ */ package org.apache.hadoop.hbase.security; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlClient.java index 922bf67a9a23..eb637517916b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlClient.java @@ -47,11 +47,12 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public class AccessControlClient { - public static final TableName ACL_TABLE_NAME = - TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "acl"); private static HTable getAclTable(Configuration conf) throws IOException { - return new HTable(conf, ACL_TABLE_NAME); + TableName aclTableName = + TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, + AccessControlConstants.OP_ATTRIBUTE_ACL); + return new HTable(conf, aclTableName.getName()); } private static BlockingInterface getAccessControlServiceStub(HTable ht) @@ -110,10 +111,12 @@ public static void grant(Configuration conf, final String namespace, public static boolean isAccessControllerRunning(Configuration conf) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { + TableName aclTableName = TableName + .valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "acl"); HBaseAdmin ha = null; try { ha = new HBaseAdmin(conf); - return ha.isTableAvailable(ACL_TABLE_NAME); + return ha.isTableAvailable(aclTableName.getNameAsString()); } finally { if (ha != null) { ha.close(); @@ -180,14 +183,16 @@ public static List getUserPermissions(Configuration conf, String Table ht = null; Admin ha = null; try { + TableName aclTableName = TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, + "acl"); ha = new HBaseAdmin(conf); - ht = new HTable(conf, ACL_TABLE_NAME); + ht = new HTable(conf, aclTableName); CoprocessorRpcChannel service = ht.coprocessorService(HConstants.EMPTY_START_ROW); BlockingInterface protocol = AccessControlProtos.AccessControlService .newBlockingStub(service); HTableDescriptor[] htds = null; - if (tableRegex == null || tableRegex.isEmpty()) { + if (tableRegex == null) { permList = ProtobufUtil.getUserPermissions(protocol); } else if (tableRegex.charAt(0) == '@') { String namespace = tableRegex.substring(1); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java index 9916a515c7a5..b190084cdca0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/Permission.java @@ -18,12 +18,7 @@ package org.apache.hadoop.hbase.security.access; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - +import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -31,7 +26,11 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.VersionedWritable; -import com.google.common.collect.Maps; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; /** * Base permissions instance representing the ability to perform a given set diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java index 87371f7f9006..a252b9e90ddd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/TablePermission.java @@ -18,17 +18,17 @@ package org.apache.hadoop.hbase.security.access; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.Bytes; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + /** * Represents an authorization for access for the given actions, optionally * restricted to the given column family or column qualifier, over the diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java index 7d44ddc28386..52323770fad3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/UserPermission.java @@ -18,16 +18,16 @@ package org.apache.hadoop.hbase.security.access; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + /** * Represents an authorization for access over the given table, column family * plus qualifier, for the given user. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java index 784aec61f108..868bd81b305a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenIdentifier.java @@ -18,17 +18,16 @@ package org.apache.hadoop.hbase.security.token; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - +import com.google.protobuf.ByteString; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.TokenIdentifier; -import com.google.protobuf.ByteString; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; /** * Represents the identity information stored in an HBase authentication token. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java index 2ce2919c6b08..bc6e678f4ab2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSelector.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hbase.security.token; -import java.util.Collection; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -28,6 +26,8 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenSelector; +import java.util.Collection; + @InterfaceAudience.Private public class AuthenticationTokenSelector implements TokenSelector { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/InvalidLabelException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/InvalidLabelException.java index d11c16728b1b..81e665ee30e1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/InvalidLabelException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/InvalidLabelException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.security.visibility; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/LabelAlreadyExistsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/LabelAlreadyExistsException.java index 3fbf93725758..bda9321f612a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/LabelAlreadyExistsException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/LabelAlreadyExistsException.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.security.visibility; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.DoNotRetryIOException; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityClient.java index 5ca5231039cf..6ac1d3f395c2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityClient.java @@ -22,12 +22,13 @@ import java.io.IOException; import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; @@ -38,7 +39,6 @@ import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import com.google.protobuf.ServiceException; @@ -92,11 +92,7 @@ public VisibilityLabelsResponse call(VisibilityLabelsService service) throws IOE } } service.addLabels(controller, builder.build(), rpcCallback); - VisibilityLabelsResponse response = rpcCallback.get(); - if (controller.failedOnException()) { - throw controller.getFailedOn(); - } - return response; + return rpcCallback.get(); } }; Map result = ht.coprocessorService( @@ -144,11 +140,7 @@ public GetAuthsResponse call(VisibilityLabelsService service) throws IOException GetAuthsRequest.Builder getAuthReqBuilder = GetAuthsRequest.newBuilder(); getAuthReqBuilder.setUser(ByteStringer.wrap(Bytes.toBytes(user))); service.getAuths(controller, getAuthReqBuilder.build(), rpcCallback); - GetAuthsResponse response = rpcCallback.get(); - if (controller.failedOnException()) { - throw controller.getFailedOn(); - } - return response; + return rpcCallback.get(); } }; Map result = ht.coprocessorService(VisibilityLabelsService.class, @@ -199,11 +191,7 @@ public VisibilityLabelsResponse call(VisibilityLabelsService service) throws IOE } else { service.clearAuths(controller, setAuthReqBuilder.build(), rpcCallback); } - VisibilityLabelsResponse response = rpcCallback.get(); - if (controller.failedOnException()) { - throw controller.getFailedOn(); - } - return response; + return rpcCallback.get(); } }; Map result = ht.coprocessorService( diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java index c51f401e009e..ce5bca051dc3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hbase.security.visibility; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Private diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java index 59ba837c6806..d439c8b0b31f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ClientSnapshotDescriptionUtils.java @@ -19,8 +19,8 @@ package org.apache.hadoop.hbase.snapshot; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java index 5d03eab28d9d..4a28461f3d2f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.snapshot; -import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.DoNotRetryIOException; /** * General exception base class for when a snapshot fails diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java index b27ff650b8dd..abeb7af0ff81 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/TablePartiallyOpenException.java @@ -17,13 +17,13 @@ */ package org.apache.hadoop.hbase.snapshot; -import java.io.IOException; - -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; +import java.io.IOException; + /** * Thrown if a table should be online/offline but is partially open */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/HasThread.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/HasThread.java index 1738a4994f12..4457fe00b332 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/HasThread.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/HasThread.java @@ -17,10 +17,10 @@ */ package org.apache.hadoop.hbase.util; -import java.lang.Thread.UncaughtExceptionHandler; - import org.apache.hadoop.hbase.classification.InterfaceAudience; +import java.lang.Thread.UncaughtExceptionHandler; + /** * Abstract class which contains a Thread and delegates the common Thread * methods to that instance. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java index 4822b0e9516f..77e2cd0ae9d5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Sleeper.java @@ -20,8 +20,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Stoppable; /** * Sleeper for current thread. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Writables.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Writables.java index e04d78902d81..15af6c96aa49 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Writables.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/Writables.java @@ -18,6 +18,10 @@ */ package org.apache.hadoop.hbase.util; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.Writable; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -26,10 +30,6 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.Writable; - /** * Utility class with methods for manipulating Writable objects */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java index 8ec3b6a67af5..b2e71e1e7c44 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java @@ -18,8 +18,17 @@ */ package org.apache.hadoop.hbase.zookeeper; -import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT; -import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.net.DNS; +import org.apache.hadoop.util.StringUtils; +import org.apache.zookeeper.server.ServerConfig; +import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; import java.io.File; import java.io.IOException; @@ -33,17 +42,8 @@ import java.util.Map.Entry; import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.util.Strings; -import org.apache.hadoop.net.DNS; -import org.apache.hadoop.util.StringUtils; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; +import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT; +import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT; /** * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java index c406aeb65871..71492e8f9e23 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java @@ -17,13 +17,10 @@ */ package org.apache.hadoop.hbase.zookeeper; -import java.io.IOException; -import java.io.InterruptedIOException; - +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; @@ -31,6 +28,9 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import java.io.IOException; +import java.io.InterruptedIOException; + /** * Manages the location of the current active Master for the RegionServer. *

diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java index 4a4dcd892ad2..98a25d327b8b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java @@ -31,11 +31,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.exceptions.DeserializationException; @@ -96,8 +96,8 @@ public boolean isLocationAvailable(ZooKeeperWatcher zkw) { */ public List> getMetaRegionsAndLocations(ZooKeeperWatcher zkw) { ServerName serverName = new MetaTableLocator().getMetaRegionLocation(zkw); - List> list = new ArrayList>(); - list.add(new Pair(HRegionInfo.FIRST_META_REGIONINFO, serverName)); + List> list = new ArrayList<>(); + list.add(new Pair<>(HRegionInfo.FIRST_META_REGIONINFO, serverName)); return list; } @@ -114,7 +114,7 @@ public List getMetaRegions(ZooKeeperWatcher zkw) { private List getListOfHRegionInfos( final List> pairs) { if (pairs == null || pairs.isEmpty()) return null; - List result = new ArrayList(pairs.size()); + List result = new ArrayList<>(pairs.size()); for (Pair pair: pairs) { result.add(pair.getFirst()); } @@ -265,7 +265,7 @@ private boolean verifyRegionLocation(AdminService.BlockingInterface hostingServe } } LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) + - " at address=" + address + ", exception=" + t.getMessage()); + " at address=" + address + ", exception=" + t); return false; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKClusterId.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKClusterId.java index b603ab274576..f0c19e339437 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKClusterId.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKClusterId.java @@ -21,9 +21,9 @@ import java.util.UUID; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.ClusterId; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.zookeeper.KeeperException; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java index 75b0175440fb..37b68b99f65d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map.Entry; @@ -27,9 +29,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.util.StringUtils; /** * Utility methods for reading, and building the ZooKeeper configuration. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java index 495c2bc0bd2f..792ed7858db7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKLeaderManager.java @@ -22,8 +22,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.zookeeper.KeeperException; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 31f273e59c96..4d4eba3696f5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -35,18 +35,18 @@ import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.RegionStoreSequenceIds; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent; @@ -801,7 +801,6 @@ private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat s * @throws KeeperException if unexpected zookeeper exception * @deprecated Unused */ - @Deprecated public static List getChildDataAndWatchForNewChildren( ZooKeeperWatcher zkw, String baseNode) throws KeeperException { List nodes = @@ -834,7 +833,6 @@ public static List getChildDataAndWatchForNewChildren( * @throws KeeperException.BadVersionException if version mismatch * @deprecated Unused */ - @Deprecated public static void updateExistingNodeData(ZooKeeperWatcher zkw, String znode, byte [] data, int expectedVersion) throws KeeperException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java index 1ed1e3fb60c6..b621200d8b1c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperNodeTracker.java @@ -20,8 +20,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Abortable; import org.apache.zookeeper.KeeperException; /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index fb98035bcf80..8d39ff645b40 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -28,11 +28,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java index 976876cf9b71..e6c2894cc127 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHColumnDescriptor.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; -import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.Test; /** Tests the HColumnDescriptor with appropriate arguments */ @Category({MiscTests.class, SmallTests.class}) @@ -51,7 +51,7 @@ public void testPb() throws DeserializationException { assertEquals(v, hcd.getMaxVersions()); hcd.setMinVersions(v); assertEquals(v, hcd.getMinVersions()); - hcd.setKeepDeletedCells(KeepDeletedCells.TRUE); + hcd.setKeepDeletedCells(!HColumnDescriptor.DEFAULT_KEEP_DELETED); hcd.setInMemory(!HColumnDescriptor.DEFAULT_IN_MEMORY); boolean inmemory = hcd.isInMemory(); hcd.setScope(v); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java index 43d94117eef3..10a99eb72d4f 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java @@ -225,39 +225,4 @@ public void testClassMethodsAreBuilderStyle() { BuilderStyleTest.assertClassesAreBuilderStyle(HTableDescriptor.class); } - - @Test - public void testModifyFamily() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table")); - byte[] familyName = Bytes.toBytes("cf"); - HColumnDescriptor hcd = new HColumnDescriptor(familyName); - hcd.setBlocksize(1000); - htd.addFamily(hcd); - assertEquals(1000, htd.getFamily(familyName).getBlocksize()); - hcd = new HColumnDescriptor(familyName); - hcd.setBlocksize(2000); - htd.modifyFamily(hcd); - assertEquals(2000, htd.getFamily(familyName).getBlocksize()); - } - - @Test(expected=IllegalArgumentException.class) - public void testModifyInexistentFamily() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table")); - byte[] familyName = Bytes.toBytes("cf"); - HColumnDescriptor hcd = new HColumnDescriptor(familyName); - htd.modifyFamily(hcd); - } - - @Test(expected=IllegalArgumentException.class) - public void testAddDuplicateFamilies() { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table")); - byte[] familyName = Bytes.toBytes("cf"); - HColumnDescriptor hcd = new HColumnDescriptor(familyName); - hcd.setBlocksize(1000); - htd.addFamily(hcd); - assertEquals(1000, htd.getFamily(familyName).getBlocksize()); - hcd = new HColumnDescriptor(familyName); - hcd.setBlocksize(2000); - htd.addFamily(hcd); - } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java index abf3da8321f0..7132b8d19db8 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java @@ -20,39 +20,20 @@ package org.apache.hadoop.hbase.client; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.junit.Assert; @@ -63,6 +44,25 @@ import org.junit.rules.Timeout; import org.mockito.Mockito; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + @Category({ClientTests.class, MediumTests.class}) public class TestAsyncProcess { private static final TableName DUMMY_TABLE = diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java index d855c743f29b..b9f1ede18992 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -42,10 +43,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.RegionLocations; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.generated.CellProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; @@ -69,9 +72,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestFastFailWithoutTestUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestFastFailWithoutTestUtil.java deleted file mode 100644 index a9f5b27331e1..000000000000 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestFastFailWithoutTestUtil.java +++ /dev/null @@ -1,613 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import static org.junit.Assert.*; - -import java.io.EOFException; -import java.io.IOException; -import java.io.SyncFailedException; -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.nio.channels.ClosedChannelException; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.exceptions.ConnectionClosingException; -import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.ipc.RemoteException; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ SmallTests.class, ClientTests.class }) -public class TestFastFailWithoutTestUtil { - private static final Log LOG = LogFactory.getLog(TestFastFailWithoutTestUtil.class); - - @Test - public void testInterceptorFactoryMethods() { - Configuration conf = HBaseConfiguration.create(); - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - RetryingCallerInterceptorFactory interceptorFactory = new RetryingCallerInterceptorFactory( - conf); - - RetryingCallerInterceptor interceptorBeforeCast = interceptorFactory - .build(); - assertTrue("We should be getting a PreemptiveFastFailInterceptor", - interceptorBeforeCast instanceof PreemptiveFastFailInterceptor); - PreemptiveFastFailInterceptor interceptor = (PreemptiveFastFailInterceptor) interceptorBeforeCast; - - RetryingCallerInterceptorContext contextBeforeCast = interceptor - .createEmptyContext(); - assertTrue( - "We should be getting a FastFailInterceptorContext since we are interacting with the" - + " PreemptiveFastFailInterceptor", - contextBeforeCast instanceof FastFailInterceptorContext); - - FastFailInterceptorContext context = (FastFailInterceptorContext) contextBeforeCast; - assertTrue(context != null); - - conf = HBaseConfiguration.create(); - interceptorFactory = new RetryingCallerInterceptorFactory(conf); - - interceptorBeforeCast = interceptorFactory.build(); - assertTrue( - "We should be getting a NoOpRetryableCallerInterceptor since we disabled PFFE", - interceptorBeforeCast instanceof NoOpRetryableCallerInterceptor); - - contextBeforeCast = interceptorBeforeCast.createEmptyContext(); - assertTrue( - "We should be getting a NoOpRetryingInterceptorContext from NoOpRetryableCallerInterceptor", - contextBeforeCast instanceof NoOpRetryingInterceptorContext); - - assertTrue(context != null); - } - - @Test - public void testInterceptorContextClear() { - PreemptiveFastFailInterceptor interceptor = createPreemptiveInterceptor(); - FastFailInterceptorContext context = (FastFailInterceptorContext) interceptor - .createEmptyContext(); - context.clear(); - assertFalse(context.getCouldNotCommunicateWithServer().booleanValue()); - assertEquals(context.didTry(), false); - assertEquals(context.getFailureInfo(), null); - assertEquals(context.getServer(), null); - assertEquals(context.getTries(), 0); - } - - @Test - public void testInterceptorContextPrepare() throws IOException { - PreemptiveFastFailInterceptor interceptor = TestFastFailWithoutTestUtil - .createPreemptiveInterceptor(); - FastFailInterceptorContext context = (FastFailInterceptorContext) interceptor - .createEmptyContext(); - RetryingCallable callable = new RegionServerCallable(null, - null, null) { - @Override - public Boolean call(int callTimeout) throws Exception { - return true; - } - - @Override - protected HRegionLocation getLocation() { - return new HRegionLocation(null, ServerName.valueOf("localhost", 1234, - 987654321)); - } - }; - context.prepare(callable); - ServerName server = getSomeServerName(); - assertEquals(context.getServer(), server); - context.clear(); - context.prepare(callable, 2); - assertEquals(context.getServer(), server); - } - - @Test - public void testInterceptorIntercept50Times() throws IOException, - InterruptedException { - for (int i = 0; i < 50; i++) { - testInterceptorIntercept(); - } - } - - public void testInterceptorIntercept() throws IOException, - InterruptedException { - Configuration conf = HBaseConfiguration.create(); - long CLEANUP_TIMEOUT = 50; - long FAST_FAIL_THRESHOLD = 10; - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS, - CLEANUP_TIMEOUT); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, - FAST_FAIL_THRESHOLD); - - PreemptiveFastFailInterceptor interceptor = TestFastFailWithoutTestUtil - .createPreemptiveInterceptor(conf); - FastFailInterceptorContext context = (FastFailInterceptorContext) interceptor - .createEmptyContext(); - - RetryingCallable callable = getDummyRetryingCallable(getSomeServerName()); - - // Lets simulate some work flow here. - int tries = 0; - context.prepare(callable, tries); - interceptor.intercept(context); - interceptor.handleFailure(context, new ConnectException( - "Failed to connect to server")); - interceptor.updateFailureInfo(context); - assertTrue("Interceptor should have updated didTry to true", - context.didTry()); - assertTrue( - "The call shouldn't have been successful if there was a ConnectException", - context.getCouldNotCommunicateWithServer().booleanValue()); - assertNull( - "Once a failure is identified, the first time the FailureInfo is generated for the server," - + " but it is not assigned to the context yet. It would be assigned on the next" - + " intercept.", context.getFailureInfo()); - assertEquals(context.getTries(), tries); - assertFalse( - "We are still in the first attempt and so we dont set this variable to true yet.", - context.isRetryDespiteFastFailMode()); - - Thread.sleep(FAST_FAIL_THRESHOLD + 1); // We sleep so as to make sure that - // we - // actually consider this server as a - // dead server in the next attempt. - tries++; - - context.prepare(callable, tries); - interceptor.intercept(context); - interceptor.handleFailure(context, new ConnectException( - "Failed to connect to server")); - interceptor.updateFailureInfo(context); - assertTrue("didTru should remain true", context.didTry()); - assertTrue( - "The call shouldn't have been successful if there was a ConnectException", - context.getCouldNotCommunicateWithServer().booleanValue()); - assertNotNull( - "The context this time is updated with a failureInfo, since we already gave it a try.", - context.getFailureInfo()); - assertEquals(context.getTries(), tries); - assertTrue( - "Since we are alone here we would be given the permission to retryDespiteFailures.", - context.isRetryDespiteFastFailMode()); - context.clear(); - - Thread.sleep(CLEANUP_TIMEOUT); // Lets try and cleanup the data in the fast - // fail failure maps. - - tries++; - - context.clear(); - context.prepare(callable, tries); - interceptor.occasionallyCleanupFailureInformation(); - assertNull("The cleanup should have cleared the server", - interceptor.repeatedFailuresMap.get(context.getServer())); - interceptor.intercept(context); - interceptor.handleFailure(context, new ConnectException( - "Failed to connect to server")); - interceptor.updateFailureInfo(context); - assertTrue("didTru should remain true", context.didTry()); - assertTrue( - "The call shouldn't have been successful if there was a ConnectException", - context.getCouldNotCommunicateWithServer().booleanValue()); - assertNull("The failureInfo is cleared off from the maps.", - context.getFailureInfo()); - assertEquals(context.getTries(), tries); - assertFalse( - "Since we are alone here we would be given the permission to retryDespiteFailures.", - context.isRetryDespiteFastFailMode()); - context.clear(); - - } - - private RetryingCallable getDummyRetryingCallable( - ServerName someServerName) { - return new RegionServerCallable(null, null, null) { - @Override - public T call(int callTimeout) throws Exception { - return null; - } - - @Override - protected HRegionLocation getLocation() { - return new HRegionLocation(null, serverName); - } - }; - } - - @Test - public void testExceptionsIdentifiedByInterceptor() throws IOException { - Throwable[] networkexceptions = new Throwable[] { - new ConnectException("Mary is unwell"), - new SocketTimeoutException("Mike is too late"), - new ClosedChannelException(), - new SyncFailedException("Dave is not on the same page"), - new TimeoutException("Mike is late again"), - new EOFException("This is the end... "), - new ConnectionClosingException("Its closing") }; - final String INDUCED = "Induced"; - Throwable[] nonNetworkExceptions = new Throwable[] { - new IOException("Bob died"), - new RemoteException("Bob's cousin died", null), - new NoSuchMethodError(INDUCED), new NullPointerException(INDUCED), - new DoNotRetryIOException(INDUCED), new Error(INDUCED) }; - - Configuration conf = HBaseConfiguration.create(); - long CLEANUP_TIMEOUT = 0; - long FAST_FAIL_THRESHOLD = 1000000; - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS, - CLEANUP_TIMEOUT); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, - FAST_FAIL_THRESHOLD); - for (Throwable e : networkexceptions) { - PreemptiveFastFailInterceptor interceptor = TestFastFailWithoutTestUtil - .createPreemptiveInterceptor(conf); - FastFailInterceptorContext context = (FastFailInterceptorContext) interceptor - .createEmptyContext(); - - RetryingCallable callable = getDummyRetryingCallable(getSomeServerName()); - context.prepare(callable, 0); - interceptor.intercept(context); - interceptor.handleFailure(context, e); - interceptor.updateFailureInfo(context); - assertTrue( - "The call shouldn't have been successful if there was a ConnectException", - context.getCouldNotCommunicateWithServer().booleanValue()); - } - for (Throwable e : nonNetworkExceptions) { - try { - PreemptiveFastFailInterceptor interceptor = TestFastFailWithoutTestUtil - .createPreemptiveInterceptor(conf); - FastFailInterceptorContext context = (FastFailInterceptorContext) interceptor - .createEmptyContext(); - - RetryingCallable callable = getDummyRetryingCallable(getSomeServerName()); - context.prepare(callable, 0); - interceptor.intercept(context); - interceptor.handleFailure(context, e); - interceptor.updateFailureInfo(context); - assertFalse( - "The call shouldn't have been successful if there was a ConnectException", - context.getCouldNotCommunicateWithServer().booleanValue()); - } catch (NoSuchMethodError t) { - assertTrue("Exception not induced", t.getMessage().contains(INDUCED)); - } catch (NullPointerException t) { - assertTrue("Exception not induced", t.getMessage().contains(INDUCED)); - } catch (DoNotRetryIOException t) { - assertTrue("Exception not induced", t.getMessage().contains(INDUCED)); - } catch (Error t) { - assertTrue("Exception not induced", t.getMessage().contains(INDUCED)); - } - } - } - - protected static PreemptiveFastFailInterceptor createPreemptiveInterceptor( - Configuration conf) { - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - RetryingCallerInterceptorFactory interceptorFactory = new RetryingCallerInterceptorFactory( - conf); - RetryingCallerInterceptor interceptorBeforeCast = interceptorFactory - .build(); - return (PreemptiveFastFailInterceptor) interceptorBeforeCast; - } - - static PreemptiveFastFailInterceptor createPreemptiveInterceptor() { - Configuration conf = HBaseConfiguration.create(); - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - return createPreemptiveInterceptor(conf); - } - - @Test(timeout = 120000) - public void testPreemptiveFastFailException50Times() - throws InterruptedException, ExecutionException { - for (int i = 0; i < 50; i++) { - testPreemptiveFastFailException(); - } - } - - /*** - * This test tries to create a thread interleaving of the 2 threads trying to do a - * Retrying operation using a {@link PreemptiveFastFailInterceptor}. The goal here is to make sure - * that the second thread will be attempting the operation while the first thread is in the - * process of making an attempt after it has marked the server in fast fail. - * - * The thread execution is as follows : - * The PreemptiveFastFailInterceptor is extended in this test to achieve a good interleaving - * behavior without using any thread sleeps. - * - * Privileged Thread 1 NonPrivileged Thread 2 - * - * Retry 0 : intercept - * - * Retry 0 : handleFailure - * latches[0].countdown - * latches2[0].await - * latches[0].await - * intercept : Retry 0 - * - * handleFailure : Retry 0 - * - * updateFailureinfo : Retry 0 - * latches2[0].countdown - * - * Retry 0 : updateFailureInfo - * - * Retry 1 : intercept - * - * Retry 1 : handleFailure - * latches[1].countdown - * latches2[1].await - * - * latches[1].await - * intercept : Retry 1 - * (throws PFFE) - * handleFailure : Retry 1 - * - * updateFailureinfo : Retry 1 - * latches2[1].countdown - * Retry 1 : updateFailureInfo - * - * - * See getInterceptor() for more details on the interceptor implementation to make sure this - * thread interleaving is achieved. - * - * We need 2 sets of latches of size MAX_RETRIES. We use an AtomicInteger done to make sure that - * we short circuit the Thread 1 after we hit the PFFE on Thread 2 - * - * - * @throws InterruptedException - * @throws ExecutionException - */ - private void testPreemptiveFastFailException() throws InterruptedException, - ExecutionException { - LOG.debug("Setting up the counters to start the test"); - priviRetryCounter.set(0); - nonPriviRetryCounter.set(0); - done.set(0); - - for (int i = 0; i <= RETRIES; i++) { - latches[i] = new CountDownLatch(1); - latches2[i] = new CountDownLatch(1); - } - - PreemptiveFastFailInterceptor interceptor = getInterceptor(); - - final RpcRetryingCaller priviCaller = getRpcRetryingCaller( - PAUSE_TIME, RETRIES, interceptor); - final RpcRetryingCaller nonPriviCaller = getRpcRetryingCaller( - PAUSE_TIME, RETRIES, interceptor); - - LOG.debug("Submitting the thread 1"); - Future priviFuture = executor.submit(new Callable() { - @Override - public Boolean call() throws Exception { - try { - isPriviThreadLocal.get().set(true); - priviCaller - .callWithRetries( - getRetryingCallable(serverName, exception), - CLEANUP_TIMEOUT); - } catch (RetriesExhaustedException e) { - return true; - } catch (PreemptiveFastFailException e) { - return false; - } - return false; - } - }); - LOG.debug("Submitting the thread 2"); - Future nonPriviFuture = executor.submit(new Callable() { - @Override - public Boolean call() throws Exception { - try { - isPriviThreadLocal.get().set(false); - nonPriviCaller.callWithRetries( - getRetryingCallable(serverName, exception), - CLEANUP_TIMEOUT); - } catch (PreemptiveFastFailException e) { - return true; - } - return false; - } - }); - LOG.debug("Waiting for Thread 2 to finish"); - assertTrue(nonPriviFuture.get()); - LOG.debug("Waiting for Thread 1 to finish"); - assertTrue(priviFuture.get()); - - // Now that the server in fast fail mode. Lets try to make contact with the - // server with a third thread. And make sure that when there is no - // exception, - // the fast fail gets cleared up. - assertTrue(interceptor.isServerInFailureMap(serverName)); - final RpcRetryingCaller priviCallerNew = getRpcRetryingCaller( - PAUSE_TIME, RETRIES, interceptor); - executor.submit(new Callable() { - @Override - public Boolean call() throws Exception { - priviCallerNew.callWithRetries( - getRetryingCallable(serverName, null), CLEANUP_TIMEOUT); - return false; - } - }).get(); - assertFalse("The server was supposed to be removed from the map", - interceptor.isServerInFailureMap(serverName)); - } - - ExecutorService executor = Executors.newCachedThreadPool(); - - /** - * Some timeouts to make the test execution resonable. - */ - final int PAUSE_TIME = 10; - final int RETRIES = 3; - final int CLEANUP_TIMEOUT = 10000; - final long FAST_FAIL_THRESHOLD = PAUSE_TIME / 1; - - /** - * The latches necessary to make the thread interleaving possible. - */ - final CountDownLatch[] latches = new CountDownLatch[RETRIES + 1]; - final CountDownLatch[] latches2 = new CountDownLatch[RETRIES + 1]; - final AtomicInteger done = new AtomicInteger(0); - - /** - * Global retry counters that give us an idea about which iteration of the retry we are in - */ - final AtomicInteger priviRetryCounter = new AtomicInteger(); - final AtomicInteger nonPriviRetryCounter = new AtomicInteger(); - final ServerName serverName = getSomeServerName(); - - /** - * The variable which is used as an identifier within the 2 threads. - */ - public final ThreadLocal isPriviThreadLocal = new ThreadLocal() { - @Override - public AtomicBoolean initialValue() { - return new AtomicBoolean(true); - } - }; - final Exception exception = new ConnectionClosingException("The current connection is closed"); - - public PreemptiveFastFailInterceptor getInterceptor() { - final Configuration conf = HBaseConfiguration.create(); - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS, - CLEANUP_TIMEOUT); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, - FAST_FAIL_THRESHOLD); - - return new PreemptiveFastFailInterceptor( - conf) { - @Override - public void updateFailureInfo(RetryingCallerInterceptorContext context) { - boolean pffe = false; - if (!isPriviThreadLocal.get().get()) { - pffe = !((FastFailInterceptorContext)context).isRetryDespiteFastFailMode(); - } - if (isPriviThreadLocal.get().get()) { - try { - // Thread 2 should be done by 2 iterations. We should short circuit Thread 1 because - // Thread 2 would be dead and can't do a countdown. - if (done.get() <= 1) { - latches2[priviRetryCounter.get()].await(); - } - } catch (InterruptedException e) { - fail(); - } - } - super.updateFailureInfo(context); - if (!isPriviThreadLocal.get().get()) { - if (pffe) done.incrementAndGet(); - latches2[nonPriviRetryCounter.get()].countDown(); - } - } - - @Override - public void intercept(RetryingCallerInterceptorContext context) - throws PreemptiveFastFailException { - if (!isPriviThreadLocal.get().get()) { - try { - latches[nonPriviRetryCounter.getAndIncrement()].await(); - } catch (InterruptedException e) { - fail(); - } - } - super.intercept(context); - } - - @Override - public void handleFailure(RetryingCallerInterceptorContext context, - Throwable t) throws IOException { - super.handleFailure(context, t); - if (isPriviThreadLocal.get().get()) { - latches[priviRetryCounter.getAndIncrement()].countDown(); - } - } - }; - } - - public RpcRetryingCaller getRpcRetryingCaller(int pauseTime, - int retries, RetryingCallerInterceptor interceptor) { - return new RpcRetryingCaller(pauseTime, retries, interceptor) { - @Override - public Void callWithRetries(RetryingCallable callable, - int callTimeout) throws IOException, RuntimeException { - Void ret = super.callWithRetries(callable, callTimeout); - return ret; - } - }; - } - - protected static ServerName getSomeServerName() { - return ServerName.valueOf("localhost", 1234, 987654321); - } - - private RegionServerCallable getRetryingCallable( - final ServerName serverName, final Exception e) { - return new RegionServerCallable(null, null, null) { - @Override - public void prepare(boolean reload) throws IOException { - this.location = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, - serverName); - } - - @Override - public Void call(int callTimeout) throws Exception { - if (e != null) - throw e; - return null; - } - - @Override - protected HRegionLocation getLocation() { - return new HRegionLocation(null, serverName); - } - - @Override - public void throwable(Throwable t, boolean retrying) { - // Do nothing - } - - @Override - public long sleep(long pause, int tries) { - return ConnectionUtils.getPauseTime(pause, tries + 1); - } - }; - } -} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java index 23e538c6893c..1617b3ac1b1a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java @@ -33,15 +33,15 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java index 96c4190dcbaf..ee65edee2c99 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java @@ -20,6 +20,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.junit.Assert; +import org.junit.Test; import java.io.IOException; import java.nio.ByteBuffer; @@ -28,10 +36,6 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; import org.apache.hadoop.hbase.filter.ColumnPaginationFilter; @@ -51,19 +55,15 @@ import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.hbase.filter.RowFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter; import org.apache.hadoop.hbase.filter.SkipFilter; import org.apache.hadoop.hbase.filter.TimestampsFilter; import org.apache.hadoop.hbase.filter.ValueFilter; import org.apache.hadoop.hbase.filter.WhileMatchFilter; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.BuilderStyleTest; import org.apache.hadoop.hbase.util.Bytes; import org.codehaus.jackson.map.ObjectMapper; -import org.junit.Assert; -import org.junit.Test; import org.junit.experimental.categories.Category; /** diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java index 129914fdcbf3..f358bf7b5d0a 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java @@ -25,11 +25,11 @@ import java.util.Arrays; import java.util.Set; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.security.visibility.Authorizations; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Test; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java index 78d718e41425..f0ed3c17f050 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java @@ -25,16 +25,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java deleted file mode 100644 index b003d5c755ce..000000000000 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestLongComparator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.filter; - -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; -import static org.junit.Assert.assertEquals; - -public class TestLongComparator { - private long values[] = { Long.MIN_VALUE, -10000000000L, -1000000L, 0L, 1000000L, 10000000000L, - Long.MAX_VALUE }; - - @Test - public void testSimple() { - for (int i = 1; i < values.length ; i++) { - for (int j = 0; j < i; j++) { - LongComparator cp = new LongComparator(values[i]); - assertEquals(1, cp.compareTo(Bytes.toBytes(values[j]))); - } - } - } -} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java index 3eab225835ee..92d67b1da8a2 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestIPCUtil.java @@ -31,13 +31,13 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.codec.Codec; import org.apache.hadoop.hbase.codec.KeyValueCodec; import org.apache.hadoop.hbase.io.SizedCellScanner; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.compress.CompressionCodec; @@ -90,7 +90,7 @@ static void doBuildCellBlockUndoCellBlock(final IPCUtil util, final Codec codec, static CellScanner getSizedCellScanner(final Cell [] cells) { int size = -1; for (Cell cell: cells) { - size += CellUtil.estimatedSerializedSizeOf(cell); + size += CellUtil.estimatedSizeOf(cell); } final int totalSize = ClassSize.align(size); final CellScanner cellScanner = CellUtil.createCellScanner(cells); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java index ed6f49b7dc3b..298344f6b6fc 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/security/TestEncryptionUtil.java @@ -17,9 +17,7 @@ */ package org.apache.hadoop.hbase.security; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.security.Key; import java.security.KeyException; @@ -28,12 +26,13 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.crypto.aes.AES; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; + import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 8a25cd440c2b..2c51cd4ef535 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -41,32 +41,13 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - + + org.apache.maven.plugins + maven-site-plugin + + true + + maven-assembly-plugin diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java index 8f299cc74455..32b4789de0c1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java @@ -235,4 +235,4 @@ public interface Cell { */ @Deprecated byte[] getRow(); -} \ No newline at end of file +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java index 9b7107b2b86f..2f635a4e9ec5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java @@ -40,25 +40,15 @@ justification="Findbugs doesn't like the way we are negating the result of a compare in below") @InterfaceAudience.Private @InterfaceStability.Evolving -public class CellComparator implements Comparator, Serializable { +public class CellComparator implements Comparator, Serializable{ private static final long serialVersionUID = -8760041766259623329L; @Override public int compare(Cell a, Cell b) { - return compare(a, b, false); + return compareStatic(a, b, false); } - /** - * Compare cells. - * TODO: Replace with dynamic rather than static comparator so can change comparator - * implementation. - * @param a - * @param b - * @param ignoreSequenceid True if we are to compare the key portion only and ignore - * the sequenceid. Set to false to compare key and consider sequenceid. - * @return 0 if equal, -1 if a < b, and +1 if a > b. - */ - public static int compare(final Cell a, final Cell b, boolean ignoreSequenceid) { + public static int compareStatic(Cell a, Cell b, boolean onlyKey) { // row int c = compareRows(a, b); if (c != 0) return c; @@ -66,7 +56,7 @@ public static int compare(final Cell a, final Cell b, boolean ignoreSequenceid) c = compareWithoutRow(a, b); if(c != 0) return c; - if (!ignoreSequenceid) { + if (!onlyKey) { // Negate following comparisons so later edits show up first // compare log replay tag value if there is any @@ -218,15 +208,8 @@ public static int compareRows(byte[] left, int loffset, int llength, byte[] righ } public static int compareWithoutRow(final Cell leftCell, final Cell rightCell) { - // If the column is not specified, the "minimum" key type appears the - // latest in the sorted order, regardless of the timestamp. This is used - // for specifying the last key/value in a given row, because there is no - // "lexicographically last column" (it would be infinitely long). The - // "maximum" key type does not need this behavior. - // Copied from KeyValue. This is bad in that we can't do memcmp w/ special rules like this. - // TODO if (leftCell.getFamilyLength() + leftCell.getQualifierLength() == 0 - && leftCell.getTypeByte() == Type.Minimum.getCode()) { + && leftCell.getTypeByte() == Type.Minimum.getCode()) { // left is "bigger", i.e. it appears later in the sorted order return 1; } @@ -402,112 +385,4 @@ public int compare(Cell a, Cell b) { } } - /** - * Try to return a Cell that falls between left and right but that is - * shorter; i.e. takes up less space. This is trick is used building HFile block index. - * Its an optimization. It does not always work. In this case we'll just return the - * right cell. - * @param left - * @param right - * @return A cell that sorts between left and right. - */ - public static Cell getMidpoint(final Cell left, final Cell right) { - // TODO: Redo so only a single pass over the arrays rather than one to compare and then a - // second composing midpoint. - if (right == null) { - throw new IllegalArgumentException("right cell can not be null"); - } - if (left == null) { - return right; - } - int diff = compareRows(left, right); - if (diff > 0) { - throw new IllegalArgumentException("Left row sorts after right row; left=" + - CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right)); - } - if (diff < 0) { - // Left row is < right row. - byte [] midRow = getMinimumMidpointArray(left.getRowArray(), left.getRowOffset(), - left.getRowLength(), - right.getRowArray(), right.getRowOffset(), right.getRowLength()); - // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; - return CellUtil.createCell(midRow); - } - // Rows are same. Compare on families. - diff = compareFamilies(left, right); - if (diff > 0) { - throw new IllegalArgumentException("Left family sorts after right family; left=" + - CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right)); - } - if (diff < 0) { - byte [] midRow = getMinimumMidpointArray(left.getFamilyArray(), left.getFamilyOffset(), - left.getFamilyLength(), - right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); - // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; - // Return new Cell where we use right row and then a mid sort family. - return CellUtil.createCell(right.getRowArray(), right.getRowOffset(), right.getRowLength(), - midRow, 0, midRow.length, HConstants.EMPTY_BYTE_ARRAY, 0, - HConstants.EMPTY_BYTE_ARRAY.length); - } - // Families are same. Compare on qualifiers. - diff = compareQualifiers(left, right); - if (diff > 0) { - throw new IllegalArgumentException("Left qualifier sorts after right qualifier; left=" + - CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right)); - } - if (diff < 0) { - byte [] midRow = getMinimumMidpointArray(left.getQualifierArray(), left.getQualifierOffset(), - left.getQualifierLength(), - right.getQualifierArray(), right.getQualifierOffset(), right.getQualifierLength()); - // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; - // Return new Cell where we use right row and family and then a mid sort qualifier. - return CellUtil.createCell(right.getRowArray(), right.getRowOffset(), right.getRowLength(), - right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength(), - midRow, 0, midRow.length); - } - // No opportunity for optimization. Just return right key. - return right; - } - - /** - * @param leftArray - * @param leftOffset - * @param leftLength - * @param rightArray - * @param rightOffset - * @param rightLength - * @return Return a new array that is between left and right and minimally sized else just return - * null as indicator that we could not create a mid point. - */ - private static byte [] getMinimumMidpointArray(final byte [] leftArray, final int leftOffset, - final int leftLength, - final byte [] rightArray, final int rightOffset, final int rightLength) { - // rows are different - int minLength = leftLength < rightLength ? leftLength : rightLength; - short diffIdx = 0; - while (diffIdx < minLength && - leftArray[leftOffset + diffIdx] == rightArray[rightOffset + diffIdx]) { - diffIdx++; - } - byte [] minimumMidpointArray = null; - if (diffIdx >= minLength) { - // leftKey's row is prefix of rightKey's. - minimumMidpointArray = new byte[diffIdx + 1]; - System.arraycopy(rightArray, rightOffset, minimumMidpointArray, 0, diffIdx + 1); - } else { - int diffByte = leftArray[leftOffset + diffIdx]; - if ((0xff & diffByte) < 0xff && (diffByte + 1) < (rightArray[rightOffset + diffIdx] & 0xff)) { - minimumMidpointArray = new byte[diffIdx + 1]; - System.arraycopy(leftArray, leftOffset, minimumMidpointArray, 0, diffIdx); - minimumMidpointArray[diffIdx] = (byte) (diffByte + 1); - } else { - minimumMidpointArray = new byte[diffIdx + 1]; - System.arraycopy(rightArray, rightOffset, minimumMidpointArray, 0, diffIdx + 1); - } - } - return minimumMidpointArray; - } -} \ No newline at end of file +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellKey.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellKey.java new file mode 100644 index 000000000000..f4c072259609 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellKey.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * This wraps the key portion of a Cell. Key includes rowkey, family, qualifier, timestamp and type + */ +@InterfaceAudience.Private +public class CellKey { + + private byte[] rowArray; + private int rowOffset; + private int rowLength; + private byte[] familyArray; + private int familyOffset; + private int familyLength; + private byte[] qualifierArray; + private int qualifierOffset; + private int qualifierLength; + private long ts; + private byte type; + + public CellKey(byte[] rowArray, int rowOffset, int rowLength, byte[] familyArray, + int familyOffset, int familyLength, byte[] qualifierArray, int qualifierOffset, + int qualifierLength, long ts, byte type) { + this.rowArray = rowArray; + this.rowOffset = rowOffset; + this.rowLength = rowLength; + this.familyArray = familyArray; + this.familyOffset = familyOffset; + this.familyLength = familyLength; + this.qualifierArray = qualifierArray; + this.qualifierOffset = qualifierOffset; + this.qualifierLength = qualifierLength; + this.ts = ts; + this.type = type; + } + + @Override + public String toString() { + String row = Bytes.toStringBinary(rowArray, rowOffset, rowLength); + String family = (familyLength == 0) ? "" : Bytes.toStringBinary(familyArray, familyOffset, + familyLength); + String qualifier = (qualifierLength == 0) ? "" : Bytes.toStringBinary(qualifierArray, + qualifierOffset, qualifierLength); + return row + "/" + family + + (family != null && family.length() > 0 ? ":" : "") + qualifier + + "/" + KeyValue.humanReadableTimestamp(ts) + "/" + Type.codeToType(type); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java index f337122b7aa1..3b5cdb9cbb40 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; /** * An interface for iterating through a sequence of cells. Similar to Java's Iterator, but without diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index beb8a7845a6c..ce0f546e627d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -26,9 +26,9 @@ import java.util.Map.Entry; import java.util.NavigableMap; -import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.ByteRange; @@ -41,11 +41,6 @@ @InterfaceStability.Evolving public final class CellUtil { - /** - * Private constructor to keep this class from being instantiated. - */ - private CellUtil(){} - /******************* ByteRange *******************************/ public static ByteRange fillRowRange(Cell cell, ByteRange range) { @@ -167,20 +162,8 @@ public static Cell createCell(final byte [] row, final byte [] family, final byt // I need a Cell Factory here. Using KeyValue for now. TODO. // TODO: Make a new Cell implementation that just carries these // byte arrays. - // TODO: Call factory to create Cell - return new KeyValue(row, family, qualifier, timestamp, KeyValue.Type.codeToType(type), value); - } - - public static Cell createCell (final byte [] rowArray, final int rowOffset, final int rowLength, - final byte [] familyArray, final int familyOffset, final int familyLength, - final byte [] qualifierArray, final int qualifierOffset, final int qualifierLength) { - // See createCell(final byte [] row, final byte [] value) for why we default Maximum type. - return new KeyValue(rowArray, rowOffset, rowLength, - familyArray, familyOffset, familyLength, - qualifierArray, qualifierOffset, qualifierLength, - HConstants.LATEST_TIMESTAMP, - KeyValue.Type.Maximum, - HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length); + return new KeyValue(row, family, qualifier, timestamp, + KeyValue.Type.codeToType(type), value); } public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, @@ -192,8 +175,7 @@ public static Cell createCell(final byte[] row, final byte[] family, final byte[ } public static Cell createCell(final byte[] row, final byte[] family, final byte[] qualifier, - final long timestamp, final byte type, final byte[] value, byte[] tags, - final long memstoreTS) { + final long timestamp, final byte type, final byte[] value, byte[] tags, final long memstoreTS) { KeyValue keyValue = new KeyValue(row, family, qualifier, timestamp, KeyValue.Type.codeToType(type), value, tags); keyValue.setSequenceId(memstoreTS); @@ -207,7 +189,7 @@ public static Cell createCell(final byte[] row, final byte[] family, final byte[ } /** - * Create a Cell with specific row. Other fields defaulted. + * Create a Cell with specific row. Other fields are arbitrary choices. * @param row * @return Cell with passed row but all other fields are arbitrary */ @@ -216,39 +198,21 @@ public static Cell createCell(final byte [] row) { } /** - * Create a Cell with specific row and value. Other fields are defaulted. + * Create a Cell with specific row and value. Other fields are arbitrary choices. * @param row * @param value * @return Cell with passed row and value but all other fields are arbitrary */ public static Cell createCell(final byte [] row, final byte [] value) { - // An empty family + empty qualifier + Type.Minimum is used as flag to indicate last on row. - // See the CellComparator and KeyValue comparator. Search for compareWithoutRow. - // Lets not make a last-on-row key as default but at same time, if you are making a key - // without specifying type, etc., flag it as weird by setting type to be Maximum. - return createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, - HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), value); - } - - /** - * Create a Cell with specific row. Other fields defaulted. - * @param row - * @param family - * @param qualifier - * @return Cell with passed row but all other fields are arbitrary - */ - public static Cell createCell(final byte [] row, final byte [] family, final byte [] qualifier) { - // See above in createCell(final byte [] row, final byte [] value) why we set type to Maximum. - return createCell(row, family, qualifier, - HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum.getCode(), HConstants.EMPTY_BYTE_ARRAY); + return createCell(row, HConstants.CATALOG_FAMILY, HConstants.SERVERNAME_QUALIFIER, + HConstants.LATEST_TIMESTAMP, (byte)0, value); } /** * @param cellScannerables * @return CellScanner interface over cellIterables */ - public static CellScanner createCellScanner( - final List cellScannerables) { + public static CellScanner createCellScanner(final List cellScannerables) { return new CellScanner() { private final Iterator iterator = cellScannerables.iterator(); private CellScanner cellScanner = null; @@ -508,14 +472,16 @@ public static boolean isDeleteColumnOrFamily(Cell cell) { * @param cell * @return Estimate of the cell size in bytes. */ - public static int estimatedSerializedSizeOf(final Cell cell) { + public static int estimatedSizeOf(final Cell cell) { // If a KeyValue, we can give a good estimate of size. if (cell instanceof KeyValue) { return ((KeyValue)cell).getLength() + Bytes.SIZEOF_INT; } // TODO: Should we add to Cell a sizeOf? Would it help? Does it make sense if Cell is // prefix encoded or compressed? - return getSumOfCellElementLengths(cell) + + return cell.getRowLength() + cell.getFamilyLength() + + cell.getQualifierLength() + + cell.getValueLength() + // Use the KeyValue's infrastructure size presuming that another implementation would have // same basic cost. KeyValue.KEY_INFRASTRUCTURE_SIZE + @@ -523,31 +489,6 @@ public static int estimatedSerializedSizeOf(final Cell cell) { Bytes.SIZEOF_INT; } - /** - * @param cell - * @return Sum of the lengths of all the elements in a Cell; does not count in any infrastructure - */ - private static int getSumOfCellElementLengths(final Cell cell) { - return getSumOfCellKeyElementLengths(cell) + cell.getValueLength() + cell.getTagsLength(); - } - - /** - * @param cell - * @return Sum of all elements that make up a key; does not include infrastructure, tags or - * values. - */ - private static int getSumOfCellKeyElementLengths(final Cell cell) { - return cell.getRowLength() + cell.getFamilyLength() + - cell.getQualifierLength() + - KeyValue.TIMESTAMP_TYPE_SIZE; - } - - public static int estimatedSerializedSizeOfKey(final Cell cell) { - if (cell instanceof KeyValue) return ((KeyValue)cell).getKeyLength(); - // This will be a low estimate. Will do for now. - return getSumOfCellKeyElementLengths(cell); - } - /** * This is an estimate of the heap space occupied by a cell. When the cell is of type * {@link HeapSize} we call {@link HeapSize#heapSize()} so cell can give a correct value. In other @@ -560,8 +501,8 @@ public static long estimatedHeapSizeOf(final Cell cell) { if (cell instanceof HeapSize) { return ((HeapSize) cell).heapSize(); } - // TODO: Add sizing of references that hold the row, family, etc., arrays. - return estimatedSerializedSizeOf(cell); + return cell.getRowLength() + cell.getFamilyLength() + cell.getQualifierLength() + + cell.getValueLength() + cell.getTagsLength() + KeyValue.TIMESTAMP_TYPE_SIZE; } /********************* tags *************************************/ @@ -692,6 +633,20 @@ public static boolean updateLatestStamp(Cell cell, byte[] ts, int tsOffset) thro return false; } + /** + * Estimation of total number of bytes used by the cell to store its key, value and tags. When the + * cell is a {@link KeyValue} we include the extra infrastructure size used by it. + * @param cell + * @return estimated length + */ + public static int estimatedLengthOf(final Cell cell) { + if (cell instanceof KeyValue) { + return ((KeyValue)cell).getLength(); + } + return cell.getRowLength() + cell.getFamilyLength() + cell.getQualifierLength() + + cell.getValueLength() + cell.getTagsLength() + KeyValue.TIMESTAMP_TYPE_SIZE; + } + /** * Writes the Cell's key part as it would have serialized in a KeyValue. The format is <2 bytes * rk len><rk><1 byte cf len><cf><qualifier><8 bytes @@ -714,43 +669,13 @@ public static void writeFlatKey(Cell cell, DataOutputStream out) throws IOExcept /** * @param cell - * @return The Key portion of the passed cell as a String. - */ - public static String getCellKeyAsString(Cell cell) { - StringBuilder sb = new StringBuilder(Bytes.toStringBinary( - cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); - sb.append('/'); - sb.append(cell.getFamilyLength() == 0? "": - Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); - // KeyValue only added ':' if family is non-null. Do same. - if (cell.getFamilyLength() > 0) sb.append(':'); - sb.append(cell.getQualifierLength() == 0? "": - Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength())); - sb.append('/'); - sb.append(KeyValue.humanReadableTimestamp(cell.getTimestamp())); - sb.append('/'); - sb.append(Type.codeToType(cell.getTypeByte())); - sb.append("/vlen="); - sb.append(cell.getValueLength()); - sb.append("/seqid="); - sb.append(cell.getSequenceId()); - return sb.toString(); - } - - /** - * This method exists just to encapsulate how we serialize keys. To be replaced by a factory - * that we query to figure what the Cell implementation is and then, what serialization engine - * to use and further, how to serialize the key for inclusion in hfile index. TODO. - * @param cell - * @return The key portion of the Cell serialized in the old-school KeyValue way or null if - * passed a null cell + * @return Key portion of the Cell including rk, cf, qualifier, ts and type. */ - public static byte [] getCellKeySerializedAsKeyValueKey(final Cell cell) { - if (cell == null) return null; - byte [] b = new byte[KeyValueUtil.keyLength(cell)]; - KeyValueUtil.appendKeyTo(cell, b, 0); - return b; + public static CellKey getCellKey(Cell cell){ + return new CellKey(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), + cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), + cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), + cell.getTimestamp(), cell.getTypeByte()); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java index 0eda1e5e4fc9..c0b358032621 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase; import java.io.DataOutput; -import java.io.IOException; +import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; @@ -29,8 +29,9 @@ import java.util.Map; import org.apache.commons.collections.iterators.UnmodifiableIterator; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes; /** @@ -72,11 +73,11 @@ private interface ImmutableConfigMap extends Iterable> int size(); } - private final List configs + protected List configs = new ArrayList(); static class ImmutableConfWrapper implements ImmutableConfigMap { - private final Configuration c; + Configuration c; ImmutableConfWrapper(Configuration conf) { c = conf; @@ -163,7 +164,7 @@ public CompoundConfiguration addBytesMap( // put new map at the front of the list (top priority) this.configs.add(0, new ImmutableConfigMap() { - private final Map m = map; + Map m = map; @Override public Iterator> iterator() { @@ -224,7 +225,7 @@ public CompoundConfiguration addStringMap(final Map map) { // put new map at the front of the list (top priority) this.configs.add(0, new ImmutableConfigMap() { - private final Map m = map; + Map m = map; @Override public Iterator> iterator() { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java index 53e9392a8365..808c4c180dbc 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java @@ -24,9 +24,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.util.VersionInfo; @@ -42,7 +42,6 @@ public class HBaseConfiguration extends Configuration { /** * Instantinating HBaseConfiguration() is deprecated. Please use * HBaseConfiguration#create() to construct a plain Configuration - * @deprecated Please use create() instead. */ @Deprecated public HBaseConfiguration() { @@ -56,7 +55,6 @@ public HBaseConfiguration() { /** * Instantiating HBaseConfiguration() is deprecated. Please use * HBaseConfiguration#create(conf) to construct a plain Configuration - * @deprecated Please user create(conf) instead. */ @Deprecated public HBaseConfiguration(final Configuration c) { @@ -169,9 +167,8 @@ public static int getInt(Configuration conf, String name, * Get the password from the Configuration instance using the * getPassword method if it exists. If not, then fall back to the * general get method for configuration elements. - * - * @param conf configuration instance for accessing the passwords - * @param alias the name of the password element + * @param conf configuration instance for accessing the passwords + * @param alias the name of the password element * @param defPass the default password * @return String password or default password * @throws IOException @@ -184,9 +181,10 @@ public static String getPassword(Configuration conf, String alias, char[] p = (char[]) m.invoke(conf, alias); if (p != null) { LOG.debug(String.format("Config option \"%s\" was found through" + - " the Configuration getPassword method.", alias)); + " the Configuration getPassword method.", alias)); passwd = new String(p); - } else { + } + else { LOG.debug(String.format( "Config option \"%s\" was not found. Using provided default value", alias)); @@ -197,7 +195,7 @@ public static String getPassword(Configuration conf, String alias, //provider API doesn't exist yet LOG.debug(String.format( "Credential.getPassword method is not available." + - " Falling back to configuration.")); + " Falling back to configuration.")); passwd = conf.get(alias, defPass); } catch (SecurityException e) { throw new IOException(e.getMessage(), e); @@ -211,8 +209,7 @@ public static String getPassword(Configuration conf, String alias, return passwd; } - /** - * For debugging. Dump configurations to system output as xml format. + /** For debugging. Dump configurations to system output as xml format. * Master and RS configurations can also be dumped using * http services. e.g. "curl http://master:16010/dump" */ diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java index 9c3367ed8337..e789b7e6bf5e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseIOException.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import java.io.IOException; + /** * All hbase specific IOExceptions should be subclasses of HBaseIOException */ diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseInterfaceAudience.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseInterfaceAudience.java index 840bbdc14da3..2acdd1c097c3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseInterfaceAudience.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseInterfaceAudience.java @@ -25,13 +25,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -public final class HBaseInterfaceAudience { - - /** - * Can't create this class. - */ - private HBaseInterfaceAudience(){} - +public class HBaseInterfaceAudience { public static final String COPROC = "Coprocesssor"; public static final String REPLICATION = "Replication"; public static final String PHOENIX = "Phoenix"; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index d4d17c9690b5..1f238285a841 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -150,9 +150,7 @@ public enum OperationStatusCode { /** Parameter name for the master type being backup (waits for primary to go inactive). */ public static final String MASTER_TYPE_BACKUP = "hbase.master.backup"; - /** - * by default every master is a possible primary master unless the conf explicitly overrides it - */ + /** by default every master is a possible primary master unless the conf explicitly overrides it */ public static final boolean DEFAULT_MASTER_TYPE_BACKUP = false; /** Name of ZooKeeper quorum configuration parameter. */ @@ -181,11 +179,8 @@ public enum OperationStatusCode { /** Default client port that the zookeeper listens on */ public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181; - /** - * Parameter name for the wait time for the recoverable zookeeper - */ - public static final String ZOOKEEPER_RECOVERABLE_WAITTIME = - "hbase.zookeeper.recoverable.waittime"; + /** Parameter name for the wait time for the recoverable zookeeper */ + public static final String ZOOKEEPER_RECOVERABLE_WAITTIME = "hbase.zookeeper.recoverable.waittime"; /** Default wait time for the recoverable zookeeper */ public static final long DEFAULT_ZOOKEPER_RECOVERABLE_WAITIME = 10000; @@ -385,10 +380,7 @@ public enum OperationStatusCode { // should go down. - /** - * The hbase:meta table's name. - * - */ + /** The hbase:meta table's name. */ @Deprecated // for compat from 0.94 -> 0.96. public static final byte[] META_TABLE_NAME = TableName.META_TABLE_NAME.getName(); @@ -1069,33 +1061,6 @@ public static enum Modify { */ public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size"; - /** - * HConstants for fast fail on the client side follow - */ - /** - * Config for enabling/disabling the fast fail mode. - */ - public static final String HBASE_CLIENT_FAST_FAIL_MODE_ENABLED = - "hbase.client.fast.fail.mode.enabled"; - - public static final boolean HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT = - false; - - public static final String HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS = - "hbase.client.fastfail.threshold"; - - public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT = - 60000; - - public static final String HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS = - "hbase.client.fast.fail.cleanup.duration"; - - public static final long HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT = - 600000; - - public static final String HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL = - "hbase.client.fast.fail.interceptor.impl"; - private HConstants() { // Can't be instantiated with this ctor. } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 22b40ec223cd..695f1f5568e0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -1890,7 +1890,7 @@ public int compareFlatKey(byte[] left, byte[] right) { } public int compareOnlyKeyPortion(Cell left, Cell right) { - return CellComparator.compare(left, right, true); + return CellComparator.compareStatic(left, right, true); } /** @@ -1899,7 +1899,7 @@ public int compareOnlyKeyPortion(Cell left, Cell right) { */ @Override public int compare(final Cell left, final Cell right) { - int compare = CellComparator.compare(left, right, false); + int compare = CellComparator.compareStatic(left, right, false); return compare; } @@ -2213,9 +2213,7 @@ public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock * @param leftKey * @param rightKey * @return 0 if equal, <0 if left smaller, >0 if right smaller - * @deprecated Since 0.99.2; Use {@link CellComparator#getMidpoint(Cell, Cell)} instead. */ - @Deprecated public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) { if (rightKey == null) { throw new IllegalArgumentException("rightKey can not be null"); @@ -2511,10 +2509,6 @@ public String getLegacyKeyComparatorName() { return "org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator"; } - /** - * @deprecated Since 0.99.2. - */ - @Deprecated public int compareFlatKey(byte[] left, int loffset, int llength, byte[] right, int roffset, int rlength) { return Bytes.BYTES_RAWCOMPARATOR.compare(left, loffset, llength, right, roffset, rlength); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index dde15bca82e4..9e969e797d65 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -24,8 +24,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java index acdaf0866d2d..31a131379ff7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java @@ -18,6 +18,10 @@ package org.apache.hadoop.hbase; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; + import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -26,10 +30,6 @@ import java.util.TreeMap; import java.util.TreeSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.util.Bytes; - /** * Namespace POJO class. Used to represent and define namespaces. * diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java index 352028a62d86..ffb347031ba3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; /** @@ -32,5 +30,5 @@ public interface SettableSequenceId { * Sets with the given seqId. * @param seqId */ - void setSequenceId(long seqId) throws IOException; + void setSequenceId(long seqId); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java index 6dac5ae0bca8..c9675c2e88cf 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; - import org.apache.hadoop.hbase.classification.InterfaceAudience; /** @@ -32,12 +30,12 @@ public interface SettableTimestamp { * Sets with the given timestamp. * @param ts */ - void setTimestamp(long ts) throws IOException; + void setTimestamp(long ts); /** * Sets with the given timestamp. * @param ts buffer containing the timestamp value * @param tsOffset offset to the new timestamp */ - void setTimestamp(byte[] ts, int tsOffset) throws IOException; + void setTimestamp(byte[] ts, int tsOffset); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java index c560a4352e6d..802319e59533 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java @@ -18,16 +18,16 @@ package org.apache.hadoop.hbase; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.util.Bytes; + import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; -import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.util.Bytes; - /** * Immutable POJO class for representing a table name. * Which is of the form: diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java index 644173c8ef54..8d3c0b93af69 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java @@ -173,26 +173,7 @@ public static List asList(byte[] b, int offset, int length) { } return tags; } - - /** - * Write a list of tags into a byte array - * @param tags - * @return the serialized tag data as bytes - */ - public static byte[] fromList(List tags) { - int length = 0; - for (Tag tag: tags) { - length += tag.length; - } - byte[] b = new byte[length]; - int pos = 0; - for (Tag tag: tags) { - System.arraycopy(tag.bytes, tag.offset, b, pos, tag.length); - pos += tag.length; - } - return b; - } - + /** * Retrieve the first tag from the tags byte array matching the passed in tag type * @param b diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java index f93d7f214a7e..f3137ae32d57 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/VersionAnnotation.java @@ -17,10 +17,7 @@ */ package org.apache.hadoop.hbase; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import java.lang.annotation.*; import org.apache.hadoop.hbase.classification.InterfaceAudience; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java index 51801a8595dc..d7f9f08319d5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java @@ -23,8 +23,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; /** * TODO javadoc @@ -63,9 +63,7 @@ private void rethrowEofException(IOException ioEx) throws IOException { LOG.trace("Error getting available for error message - ignoring", t); } if (!isEof) throw ioEx; - if (LOG.isTraceEnabled()) { - LOG.trace("Partial cell read caused by EOF", ioEx); - } + LOG.error("Partial cell read caused by EOF: " + ioEx); EOFException eofEx = new EOFException("Partial cell read"); eofEx.initCause(ioEx); throw eofEx; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java index 7a96abe5cf27..6f2231c9d4fa 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseEncoder.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.io.OutputStream; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; /** * TODO javadoc diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java index 9d03d89a775c..77cf80a969af 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java @@ -22,10 +22,10 @@ import java.io.OutputStream; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecWithTags.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecWithTags.java index a614026c1a94..b4efaf803918 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecWithTags.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecWithTags.java @@ -22,10 +22,10 @@ import java.io.OutputStream; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/Codec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/Codec.java index 7fc1749fc711..7801e3b88103 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/Codec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/Codec.java @@ -20,9 +20,9 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.CellOutputStream; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java index 8ebe25a53aa0..440ae781f551 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CodecException.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.codec; -import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HBaseIOException; /** * Thrown when problems in the codec whether setup or context. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java index f41d6b01fd68..cfe974219813 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java @@ -21,11 +21,11 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * Codec that does KeyValue version 1 serialization. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java index 664fcac9cb75..02158f432c0e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java @@ -21,11 +21,11 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * Codec that does KeyValue version 1 serialization with serializing tags also. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferInputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferInputStream.java deleted file mode 100644 index 1530ccd3185f..000000000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBufferInputStream.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io; - -import java.io.InputStream; -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; - -/** - * Not thread safe! - *

- * Please note that the reads will cause position movement on wrapped ByteBuffer. - */ -@InterfaceAudience.Private -public class ByteBufferInputStream extends InputStream { - - private ByteBuffer buf; - - public ByteBufferInputStream(ByteBuffer buf) { - this.buf = buf; - } - - /** - * Reads the next byte of data from this input stream. The value byte is returned as an - * int in the range 0 to 255. If no byte is available - * because the end of the stream has been reached, the value -1 is returned. - * @return the next byte of data, or -1 if the end of the stream has been reached. - */ - public int read() { - if (this.buf.hasRemaining()) { - return (this.buf.get() & 0xff); - } - return -1; - } - - /** - * Reads up to next len bytes of data from buffer into passed array(starting from - * given offset). - * @param b the array into which the data is read. - * @param off the start offset in the destination array b - * @param len the maximum number of bytes to read. - * @return the total number of bytes actually read into the buffer, or -1 if not even - * 1 byte can be read because the end of the stream has been reached. - */ - public int read(byte b[], int off, int len) { - int avail = available(); - if (avail <= 0) { - return -1; - } - - if (len > avail) { - len = avail; - } - if (len <= 0) { - return 0; - } - - this.buf.get(b, off, len); - return len; - } - - /** - * Skips n bytes of input from this input stream. Fewer bytes might be skipped if the - * end of the input stream is reached. The actual number k of bytes to be skipped is - * equal to the smaller of n and remaining bytes in the stream. - * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. - */ - public long skip(long n) { - long k = Math.min(n, available()); - if (k < 0) { - k = 0; - } - this.buf.position((int) (this.buf.position() + k)); - return k; - } - - /** - * @return the number of remaining bytes that can be read (or skipped - * over) from this input stream. - */ - public int available() { - return this.buf.remaining(); - } -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/CellOutputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/CellOutputStream.java index 75429e52e4be..4750413fa0db 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/CellOutputStream.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/CellOutputStream.java @@ -20,10 +20,10 @@ import java.io.IOException; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; /** * Accepts a stream of Cells. This can be used to build a block of cells during compactions diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java index 73525c5a7a25..b70e94a15acb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/ImmutableBytesWritable.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hbase.io; +import java.io.IOException; import java.io.DataInput; import java.io.DataOutput; -import java.io.IOException; import java.util.Arrays; import java.util.List; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java index a9fdd2145625..8a21f9edda13 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java @@ -19,13 +19,13 @@ package org.apache.hadoop.hbase.io; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + /** * Copied from guava source code v15 (LimitedInputStream) * Guava deprecated LimitInputStream in v14 and removed it in v15. Copying this class here diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/SizedCellScanner.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/SizedCellScanner.java index ed272efa096b..9fc7033dd866 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/SizedCellScanner.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/SizedCellScanner.java @@ -16,10 +16,11 @@ * limitations under the License. */ package org.apache.hadoop.hbase.io; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.CellScanner; + /** * A CellScanner that knows its size in memory in bytes. * Used playing the CellScanner into an in-memory buffer; knowing the size ahead of time saves diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java index 26e7b507c52e..537437d5630e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TagCompressionContext.java @@ -25,8 +25,8 @@ import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.io.util.Dictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.ByteBufferUtils; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java index edb4dfa4d38a..8a349db3be61 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java @@ -25,10 +25,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.compress.CodecPool; import org.apache.hadoop.io.compress.CompressionCodec; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CipherProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CipherProvider.java index 5a475ccfd4d4..6deb36517449 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CipherProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CipherProvider.java @@ -16,9 +16,9 @@ */ package org.apache.hadoop.hbase.io.crypto; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; /** * An CipherProvider contributes support for various cryptographic diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Context.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Context.java index 1e2881e754d5..31cca0e98e8a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Context.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Context.java @@ -18,11 +18,11 @@ import java.security.Key; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.MD5Hash; import com.google.common.base.Preconditions; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/DefaultCipherProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/DefaultCipherProvider.java index 4f2aebe844b8..961fbae5f6d3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/DefaultCipherProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/DefaultCipherProvider.java @@ -16,10 +16,10 @@ */ package org.apache.hadoop.hbase.io.crypto; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.io.crypto.aes.AES; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java index 2a3c33791beb..85d6eb59bd11 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Encryption.java @@ -35,11 +35,11 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.ReflectionUtils; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/KeyStoreKeyProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/KeyStoreKeyProvider.java index 62167d6f84e1..0e5f36e21737 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/KeyStoreKeyProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/KeyStoreKeyProvider.java @@ -19,8 +19,8 @@ import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; import java.io.InputStream; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index be8c1927739d..a4b38571d1b9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -21,17 +21,17 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.SettableSequenceId; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.SettableSequenceId; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.hfile.BlockType; @@ -137,7 +137,7 @@ protected void ensureSpaceForTags() { } } - protected void setKey(byte[] keyBuffer, long memTS) { + protected void createKeyOnlyKeyValue(byte[] keyBuffer, long memTS) { currentKey.setKey(keyBuffer, 0, keyLength); memstoreTS = memTS; } @@ -302,8 +302,11 @@ public byte[] getRow() { @Override public String toString() { - return KeyValue.keyToString(this.keyBuffer, 0, KeyValueUtil.keyLength(this)) + "/vlen=" - + getValueLength() + "/seqid=" + memstoreTS; + KeyValue kv = KeyValueUtil.copyToNewKeyValue(this); + if (kv == null) { + return "null"; + } + return kv.toString(); } public Cell shallowCopy() { @@ -328,7 +331,7 @@ public Cell shallowCopy() { protected static class ClonedSeekerState implements Cell, HeapSize, SettableSequenceId { private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + (4 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (7 * Bytes.SIZEOF_INT) - + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (2 * ClassSize.ARRAY)); + + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE)); private byte[] keyOnlyBuffer; private ByteBuffer currentBuffer; private short rowLength; @@ -512,7 +515,8 @@ public void setSequenceId(long seqId) { @Override public long heapSize() { - return FIXED_OVERHEAD + rowLength + familyLength + qualifierLength + valueLength + tagsLength; + return FIXED_OVERHEAD + rowLength + familyLength + qualifierLength + valueLength + tagsLength + + KeyValue.TIMESTAMP_TYPE_SIZE; } } @@ -572,7 +576,7 @@ public void setCurrentBuffer(ByteBuffer buffer) { current.tagCompressionContext = tagCompressionContext; } decodeFirst(); - current.setKey(current.keyBuffer, current.memstoreTS); + current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS); previous.invalidate(); } @@ -585,10 +589,9 @@ public ByteBuffer getKeyDeepCopy() { @Override public ByteBuffer getValueShallowCopy() { - ByteBuffer dup = currentBuffer.duplicate(); - dup.position(current.valueOffset); - dup.limit(current.valueOffset + current.valueLength); - return dup.slice(); + return ByteBuffer.wrap(currentBuffer.array(), + currentBuffer.arrayOffset() + current.valueOffset, + current.valueLength); } @Override @@ -597,7 +600,8 @@ public ByteBuffer getKeyValueBuffer() { kvBuffer.putInt(current.keyLength); kvBuffer.putInt(current.valueLength); kvBuffer.put(current.keyBuffer, 0, current.keyLength); - ByteBufferUtils.copyFromBufferToBuffer(kvBuffer, currentBuffer, current.valueOffset, + kvBuffer.put(currentBuffer.array(), + currentBuffer.arrayOffset() + current.valueOffset, current.valueLength); if (current.tagsLength > 0) { // Put short as unsigned @@ -606,7 +610,7 @@ public ByteBuffer getKeyValueBuffer() { if (current.tagsOffset != -1) { // the offset of the tags bytes in the underlying buffer is marked. So the temp // buffer,tagsBuffer was not been used. - ByteBufferUtils.copyFromBufferToBuffer(kvBuffer, currentBuffer, current.tagsOffset, + kvBuffer.put(currentBuffer.array(), currentBuffer.arrayOffset() + current.tagsOffset, current.tagsLength); } else { // When tagsOffset is marked as -1, tag compression was present and so the tags were @@ -636,7 +640,7 @@ public void rewind() { tagCompressionContext.clear(); } decodeFirst(); - current.setKey(current.keyBuffer, current.memstoreTS); + current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS); previous.invalidate(); } @@ -646,7 +650,7 @@ public boolean next() { return false; } decodeNext(); - current.setKey(current.keyBuffer, current.memstoreTS); + current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS); previous.invalidate(); return true; } @@ -669,7 +673,7 @@ protected void decodeTags() { } current.tagsOffset = -1; } else { - // When tag compress is not used, let us not do copying of tags bytes into tagsBuffer. + // When tag compress is not used, let us not do temp copying of tags bytes into tagsBuffer. // Just mark the tags Offset so as to create the KV buffer later in getKeyValueBuffer() current.tagsOffset = currentBuffer.position(); ByteBufferUtils.skip(currentBuffer, current.tagsLength); @@ -780,7 +784,7 @@ public int seekToKeyInBlock(Cell seekCell, boolean seekBefore) { if (currentBuffer.hasRemaining()) { previous.copyFromNext(current); decodeNext(); - current.setKey(current.keyBuffer, current.memstoreTS); + current.createKeyOnlyKeyValue(current.keyBuffer, current.memstoreTS); } else { break; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java index 5ac5d2ed1476..48ae78833129 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CompressionState.java @@ -18,8 +18,8 @@ import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.ByteBufferUtils; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java index 6b87c7740a0c..3baebc2c6239 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; @@ -68,13 +68,11 @@ public int internalEncode(Cell cell, HFileBlockDefaultEncodingContext encodingCo @Override public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { int keyLength = block.getInt(Bytes.SIZEOF_INT); - ByteBuffer dup = block.duplicate(); - int pos = 3 * Bytes.SIZEOF_INT; - dup.position(pos); - dup.limit(pos + keyLength); - return dup.slice(); + return ByteBuffer.wrap(block.array(), + block.arrayOffset() + 3 * Bytes.SIZEOF_INT, keyLength).slice(); } + @Override public String toString() { return CopyKeyDataBlockEncoder.class.getSimpleName(); @@ -125,4 +123,5 @@ protected ByteBuffer internalDecodeKeyValues(DataInputStream source, int allocat return buffer; } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index 872c22c54909..0e85380fd11d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -21,9 +21,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.hfile.HFileContext; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index 4182dc45ea4d..c39822fb1bd5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -317,7 +317,6 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { ByteBuffer result = ByteBuffer.allocate(keyLength); // copy row - assert !(result.isDirect()); int pos = result.arrayOffset(); block.get(result.array(), pos, Bytes.SIZEOF_SHORT); pos += Bytes.SIZEOF_SHORT; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java index 1f7f8fa1f65c..d83c0b782fc0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java @@ -26,10 +26,10 @@ import java.util.Iterator; import org.apache.commons.lang.NotImplementedException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.util.ByteBufferUtils; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodingState.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodingState.java index a333a15cb0df..3adc5487d0ef 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodingState.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodingState.java @@ -18,8 +18,8 @@ */ package org.apache.hadoop.hbase.io.encoding; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; /** * Keeps track of the encoding state. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index a6f43d01ada9..94069bffaff5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -362,10 +362,8 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { ByteBufferUtils.readCompressedInt(block); // commonLength int pos = block.position(); block.reset(); - ByteBuffer dup = block.duplicate(); - dup.position(pos); - dup.limit(pos + keyLength); - return dup.slice(); + return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength) + .slice(); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java index 37001cc34beb..5889c7b54781 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java @@ -40,14 +40,16 @@ public interface HFileBlockDecodingContext { * @param uncompressedSizeWithoutHeader numBytes without header required to store the block after * decompressing (not decoding) * @param blockBufferWithoutHeader ByteBuffer pointed after the header but before the data - * @param onDiskBlock on disk data to be decoded + * @param onDiskBlock on disk bytes to be decoded + * @param offset data start offset in onDiskBlock * @throws IOException */ void prepareDecoding( int onDiskSizeWithoutHeader, int uncompressedSizeWithoutHeader, ByteBuffer blockBufferWithoutHeader, - ByteBuffer onDiskBlock + byte[] onDiskBlock, + int offset ) throws IOException; /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java index 78bb0d6af0d9..b27a998babc4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hbase.io.encoding; +import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -23,13 +24,13 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.ByteBufferInputStream; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.Decryptor; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.Bytes; /** @@ -51,8 +52,9 @@ public HFileBlockDefaultDecodingContext(HFileContext fileContext) { @Override public void prepareDecoding(int onDiskSizeWithoutHeader, int uncompressedSizeWithoutHeader, - ByteBuffer blockBufferWithoutHeader, ByteBuffer onDiskBlock) throws IOException { - InputStream in = new DataInputStream(new ByteBufferInputStream(onDiskBlock)); + ByteBuffer blockBufferWithoutHeader, byte[] onDiskBlock, int offset) throws IOException { + InputStream in = new DataInputStream(new ByteArrayInputStream(onDiskBlock, offset, + onDiskSizeWithoutHeader)); Encryption.Context cryptoContext = fileContext.getEncryptionContext(); if (cryptoContext != Encryption.Context.NONE) { @@ -85,7 +87,6 @@ public void prepareDecoding(int onDiskSizeWithoutHeader, int uncompressedSizeWit } Compression.Algorithm compression = fileContext.getCompression(); - assert blockBufferWithoutHeader.hasArray(); if (compression != Compression.Algorithm.NONE) { Compression.decompress(blockBufferWithoutHeader.array(), blockBufferWithoutHeader.arrayOffset(), in, onDiskSizeWithoutHeader, diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java index 0286eca7c8d4..8ff71a950545 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java @@ -21,12 +21,12 @@ import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -184,10 +184,8 @@ public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { } int pos = block.position(); block.reset(); - ByteBuffer dup = block.duplicate(); - dup.position(pos); - dup.limit(pos + keyLength); - return dup.slice(); + return ByteBuffer.wrap(block.array(), block.arrayOffset() + pos, keyLength) + .slice(); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java index 0db584ea2148..2583647b1184 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java @@ -162,10 +162,12 @@ public static BlockType read(DataInputStream in) throws IOException { } public static BlockType read(ByteBuffer buf) throws IOException { - byte[] magicBuf = new byte[Math.min(buf.limit() - buf.position(), MAGIC_LENGTH)]; - buf.get(magicBuf); - BlockType blockType = parse(magicBuf, 0, magicBuf.length); + BlockType blockType = parse(buf.array(), + buf.arrayOffset() + buf.position(), + Math.min(buf.limit() - buf.position(), MAGIC_LENGTH)); + // If we got here, we have read exactly MAGIC_LENGTH bytes. + buf.position(buf.position() + MAGIC_LENGTH); return blockType; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java index ce8b71a307b4..83fe70108cfe 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java @@ -16,8 +16,8 @@ * limitations under the License. */ package org.apache.hadoop.hbase.io.hfile; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java index 5c5d75fe6ce7..9a4234a8405b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase.io.hfile; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java index 250a984ed193..f464db99d5ec 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.classification.InterfaceAudience; @InterfaceAudience.Private public class HeapMemorySizeUtil { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java index 5f316f23252d..901c57c3e260 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/User.java @@ -24,8 +24,8 @@ import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Methods; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java index f2a16caa7ef5..82f686f9120c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/security/UserProvider.java @@ -19,9 +19,9 @@ import java.io.IOException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.BaseConfigurable; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/PBType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/PBType.java index ecc9c802bd68..91e952476c0d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/PBType.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/PBType.java @@ -17,12 +17,11 @@ */ package org.apache.hadoop.hbase.types; -import org.apache.hadoop.hbase.util.Order; -import org.apache.hadoop.hbase.util.PositionedByteRange; - import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.Message; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; /** * A base-class for {@link DataType} implementations backed by protobuf. See diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Base64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Base64.java index 76e2549afdc5..4f31861c299a 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Base64.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Base64.java @@ -19,6 +19,11 @@ package org.apache.hadoop.hbase.util; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -38,11 +43,6 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - /** * Encodes and decodes to and from Base64 notation. * diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java index 26899469a8fe..b4c6690e7adb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java @@ -331,10 +331,7 @@ public static ByteBuffer drainInputStreamToBuffer(InputStream is) throws IOExcep } /** - * Copy from one buffer to another from given offset. - *

- * Note : This will advance the position marker of {@code out} but not change the position maker - * for {@code in} + * Copy from one buffer to another from given offset * @param out destination buffer * @param in source buffer * @param sourceOffset offset in the source buffer @@ -354,27 +351,6 @@ public static void copyFromBufferToBuffer(ByteBuffer out, } } - /** - * Copy from one buffer to another from given offset. This will be absolute positional copying and - * won't affect the position of any of the buffers. - * @param out - * @param in - * @param sourceOffset - * @param destinationOffset - * @param length - */ - public static void copyFromBufferToBuffer(ByteBuffer out, ByteBuffer in, int sourceOffset, - int destinationOffset, int length) { - if (in.hasArray() && out.hasArray()) { - System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), out.arrayOffset() - + destinationOffset, length); - } else { - for (int i = 0; i < length; ++i) { - out.put((destinationOffset + i), in.get(sourceOffset + i)); - } - } - } - /** * Find length of common prefix of two parts in the buffer * @param buffer Where parts are located. @@ -478,35 +454,4 @@ public static byte[] toBytes(ByteBuffer buffer, int startPosition) { return output; } - /** - * Copy the given number of bytes from specified offset into a new byte[] - * @param buffer - * @param offset - * @param length - * @return a new byte[] containing the bytes in the specified range - */ - public static byte[] toBytes(ByteBuffer buffer, int offset, int length) { - byte[] output = new byte[length]; - for (int i = 0; i < length; i++) { - output[i] = buffer.get(offset + i); - } - return output; - } - - public static int compareTo(ByteBuffer buf1, int o1, int len1, ByteBuffer buf2, int o2, int len2) { - if (buf1.hasArray() && buf2.hasArray()) { - return Bytes.compareTo(buf1.array(), buf1.arrayOffset() + o1, len1, buf2.array(), - buf2.arrayOffset() + o2, len2); - } - int end1 = o1 + len1; - int end2 = o2 + len2; - for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) { - int a = buf1.get(i) & 0xFF; - int b = buf2.get(j) & 0xFF; - if (a != b) { - return a - b; - } - } - return len1 - len2; - } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java index cc09c3ae3e44..4fc09e103a61 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ChecksumFactory.java @@ -20,8 +20,9 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; -import java.lang.reflect.Constructor; +import java.lang.ClassNotFoundException; import java.util.zip.Checksum; +import java.lang.reflect.Constructor; /** * Utility class that is used to generate a Checksum object. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcatenatedLists.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcatenatedLists.java index 2180b4d39272..cf8c191d0c0e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcatenatedLists.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcatenatedLists.java @@ -21,6 +21,7 @@ import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentIndex.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentIndex.java index 539f33b01022..af39d1158a30 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentIndex.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ConcurrentIndex.java @@ -20,6 +20,9 @@ package org.apache.hadoop.hbase.util; +import com.google.common.base.Supplier; +import com.google.common.collect.Multiset; + import java.util.Comparator; import java.util.ConcurrentModificationException; import java.util.Set; @@ -30,9 +33,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import com.google.common.base.Supplier; -import com.google.common.collect.Multiset; - /** * A simple concurrent map of sets. This is similar in concept to * {@link Multiset}, with the following exceptions: diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java index a3aabf7686bf..7b837afb2d24 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java @@ -35,10 +35,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.io.IOUtils; import com.google.common.base.Preconditions; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java index 81e4483ec9c8..e4345588e72d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/DynamicClassLoader.java @@ -25,11 +25,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * This is a class loader that can load classes dynamically from new diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Hash.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Hash.java index 82cf5c425535..a7d58434b9c7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Hash.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Hash.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.util; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; /** * This class represents a common API for hashing functions. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java index 0e50dfbd0b09..cf11de17da2c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java @@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.util; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.HConstants; @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java index d02c5e992b48..f21ff3186eb3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/VersionInfo.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hbase.util; +import org.apache.commons.logging.LogFactory; import java.io.PrintWriter; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.VersionAnnotation; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.VersionAnnotation; +import org.apache.commons.logging.Log; /** * This class finds the package info for hbase and the VersionAnnotation @@ -93,7 +93,7 @@ public static String getUrl() { static String[] versionReport() { return new String[] { "HBase " + getVersion(), - "Source code repository " + getUrl() + " revision=" + getRevision(), + "Source code repository " + getUrl() + " -r " + getRevision(), "Compiled by " + getUser() + " on " + getDate(), "From source with checksum " + getSrcChecksum() }; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java index 52bc4e0999c7..859c07b5e018 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java @@ -24,9 +24,9 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.io.WritableUtils; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java index 5d4d94177198..18368a4a214d 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ClassTestFinder.java @@ -22,6 +22,8 @@ import java.lang.reflect.Modifier; import java.util.regex.Pattern; +import org.apache.hadoop.hbase.ClassFinder.ClassFilter; +import org.apache.hadoop.hbase.ClassFinder.FileNameFilter; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runners.Suite; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java index 3cae4d2fb7fb..18ca35c11b51 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java @@ -25,10 +25,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; /** * Common helpers for testing HBase that do not depend on specific server/etc. things. diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java index 539aea3ab4e8..e56dea8669d0 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java @@ -19,12 +19,11 @@ package org.apache.hadoop.hbase; -import java.util.ArrayList; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.util.*; + /** * Utility class to check the resources: * - log them before and after each test method diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java index 6264a5ef4108..26f63f892181 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitListener.java @@ -29,9 +29,10 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hbase.ResourceChecker.Phase; -import org.apache.hadoop.hbase.util.JVM; import org.junit.runner.notification.RunListener; +import org.apache.hadoop.hbase.util.JVM; + /** * Listen to the test progress and check the usage of: * - threads diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java index d6a2f7286457..2b80a546796f 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.apache.hadoop.hbase.KeyValue.Type; @@ -46,7 +45,7 @@ public class TestCellComparator { public void testCompareCells() { KeyValue kv1 = new KeyValue(row1, fam1, qual1, val); KeyValue kv2 = new KeyValue(row2, fam1, qual1, val); - assertTrue((CellComparator.compare(kv1, kv2, false)) < 0); + assertTrue((CellComparator.compareStatic(kv1, kv2, false)) < 0); kv1 = new KeyValue(row1, fam2, qual1, val); kv2 = new KeyValue(row1, fam1, qual1, val); @@ -54,11 +53,11 @@ public void testCompareCells() { kv1 = new KeyValue(row1, fam1, qual1, 1l, val); kv2 = new KeyValue(row1, fam1, qual1, 2l, val); - assertTrue((CellComparator.compare(kv1, kv2, false) > 0)); + assertTrue((CellComparator.compareStatic(kv1, kv2, false) > 0)); kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Maximum); - assertTrue((CellComparator.compare(kv1, kv2, false) > 0)); + assertTrue((CellComparator.compareStatic(kv1, kv2, false) > 0)); kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); kv2 = new KeyValue(row1, fam_1_2, qual1, 1l, Type.Maximum); @@ -76,59 +75,4 @@ public void testCompareCells() { kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); assertTrue((CellComparator.equals(kv1, kv2))); } - - @Test - public void testGetShortMidpoint() { - Cell left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - Cell right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - Cell mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) <= 0); - assertTrue(CellComparator.compare(mid, right, true) <= 0); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("a")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) <= 0); - - left = CellUtil.createCell(Bytes.toBytes("g"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("i"), Bytes.toBytes("a"), Bytes.toBytes("a")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) <= 0); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("bbbbbbb"), Bytes.toBytes("a"), Bytes.toBytes("a")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) < 0); - assertEquals(1, (int)mid.getRowLength()); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) <= 0); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaa"), Bytes.toBytes("b")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) < 0); - assertEquals(2, (int)mid.getFamilyLength()); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) < 0); - assertEquals(2, (int)mid.getQualifierLength()); - - left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); - right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); - mid = CellComparator.getMidpoint(left, right); - assertTrue(CellComparator.compare(left, mid, true) < 0); - assertTrue(CellComparator.compare(mid, right, true) <= 0); - assertEquals(1, (int)mid.getQualifierLength()); - } -} \ No newline at end of file +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java index 5c18b519c992..182c4dbe4b14 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hbase; -import static org.junit.Assert.*; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -372,29 +370,4 @@ public void testFindCommonPrefixInFlatKey() { + kv6.getFamilyLength() + kv6.getQualifierLength(), CellUtil.findCommonPrefixInFlatKey(kv6, kv8, true, false)); } - - /** - * Assert CellUtil makes Cell toStrings same way we do KeyValue toStrings. - */ - @Test - public void testToString() { - byte [] row = Bytes.toBytes("row"); - long ts = 123l; - // Make a KeyValue and a Cell and see if same toString result. - KeyValue kv = new KeyValue(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, - ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY); - Cell cell = CellUtil.createCell(row, HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, - ts, KeyValue.Type.Minimum.getCode(), HConstants.EMPTY_BYTE_ARRAY); - String cellToString = CellUtil.getCellKeyAsString(cell); - assertEquals(kv.toString(), cellToString); - // Do another w/ non-null family. - byte [] f = new byte [] {'f'}; - byte [] q = new byte [] {'q'}; - kv = new KeyValue(row, f, q, ts, KeyValue.Type.Minimum, HConstants.EMPTY_BYTE_ARRAY); - cell = CellUtil.createCell(row, f, q, ts, KeyValue.Type.Minimum.getCode(), - HConstants.EMPTY_BYTE_ARRAY); - cellToString = CellUtil.getCellKeyAsString(cell); - assertEquals(kv.toString(), cellToString); - - } -} \ No newline at end of file +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/Waiter.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/Waiter.java index 3453baf13b78..468333ddd395 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/Waiter.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/Waiter.java @@ -25,8 +25,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; /** * A class that provides a standard waitFor pattern diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java index 922de6f7f48e..028e78e2b4fe 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java @@ -17,9 +17,7 @@ */ package org.apache.hadoop.hbase.codec; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecWithTags.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecWithTags.java index 30f2f00fa48a..14926f1fc8f1 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecWithTags.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecWithTags.java @@ -32,9 +32,9 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestKeyValueCodecWithTags.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestKeyValueCodecWithTags.java index 007647a04b48..78fa733a7d03 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestKeyValueCodecWithTags.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestKeyValueCodecWithTags.java @@ -32,9 +32,9 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBufferInputStream.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBufferInputStream.java deleted file mode 100644 index 30fb71ee9540..000000000000 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBufferInputStream.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io; - -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.testclassification.IOTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({ IOTests.class, SmallTests.class }) -public class TestByteBufferInputStream { - - @Test - public void testReads() throws Exception { - ByteArrayOutputStream bos = new ByteArrayOutputStream(100); - DataOutputStream dos = new DataOutputStream(bos); - String s = "test"; - int i = 128; - dos.write(1); - dos.writeInt(i); - dos.writeBytes(s); - dos.writeLong(12345L); - dos.writeShort(2); - dos.flush(); - ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray()); - - // bbis contains 19 bytes - // 1 byte, 4 bytes int, 4 bytes string, 8 bytes long and 2 bytes short - ByteBufferInputStream bbis = new ByteBufferInputStream(bb); - assertEquals(15 + s.length(), bbis.available()); - assertEquals(1, bbis.read()); - byte[] ib = new byte[4]; - bbis.read(ib); - assertEquals(i, Bytes.toInt(ib)); - byte[] sb = new byte[s.length()]; - bbis.read(sb); - assertEquals(s, Bytes.toString(sb)); - byte[] lb = new byte[8]; - bbis.read(lb); - assertEquals(12345, Bytes.toLong(lb)); - assertEquals(2, bbis.available()); - ib = new byte[4]; - int read = bbis.read(ib, 0, ib.length); - // We dont have 4 bytes remainig but only 2. So onlt those should be returned back - assertEquals(2, read); - assertEquals(2, Bytes.toShort(ib)); - assertEquals(0, bbis.available()); - // At end. The read() should return -1 - assertEquals(-1, bbis.read()); - bbis.close(); - - bb = ByteBuffer.wrap(bos.toByteArray()); - bbis = new ByteBufferInputStream(bb); - DataInputStream dis = new DataInputStream(bbis); - dis.read(); - assertEquals(i, dis.readInt()); - dis.close(); - } -} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java index 841c4681a4cb..fcdb750435a4 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java @@ -27,10 +27,10 @@ import java.util.List; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java index 781924b22b0c..9b45d0996ada 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hbase.io.crypto; import java.security.Key; - import javax.crypto.spec.SecretKeySpec; /** diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java index fdb9448c1df6..b7d6322c7e53 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestCipherProvider.java @@ -16,9 +16,7 @@ */ package org.apache.hadoop.hbase.io.crypto; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.IOException; import java.io.InputStream; @@ -29,9 +27,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.io.crypto.aes.AES; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.io.crypto.aes.AES; + import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java index d36333efef26..5a7222d30101 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestEncryption.java @@ -16,8 +16,7 @@ */ package org.apache.hadoop.hbase.io.crypto; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java index dab03f22e623..4e60082297f0 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java @@ -25,9 +25,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.io.crypto.aes.AES; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.io.crypto.aes.AES; + import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestAES.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestAES.java index ea8879b3b0ff..68424bb98adc 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestAES.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestAES.java @@ -16,9 +16,7 @@ */ package org.apache.hadoop.hbase.io.crypto.aes; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -36,13 +34,14 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.io.crypto.Cipher; import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.Encryptor; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; + import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestLRUDictionary.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestLRUDictionary.java index 9569ba85d8b9..90da280ebde5 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestLRUDictionary.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/util/TestLRUDictionary.java @@ -18,9 +18,7 @@ package org.apache.hadoop.hbase.io.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import java.math.BigInteger; import java.util.Arrays; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java index 09ef707b58a6..bd4854ea69d6 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBase64.java @@ -24,7 +24,6 @@ import java.util.TreeMap; import junit.framework.TestCase; - import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRangeWithKVSerialization.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRangeWithKVSerialization.java index a6b7cc52cb18..34326d67588d 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRangeWithKVSerialization.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRangeWithKVSerialization.java @@ -21,9 +21,9 @@ import java.util.List; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.Tag; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestConcatenatedLists.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestConcatenatedLists.java index fd4baf58a993..17d54937e074 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestConcatenatedLists.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestConcatenatedLists.java @@ -18,11 +18,6 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -34,6 +29,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.*; + @Category({MiscTests.class, SmallTests.class}) public class TestConcatenatedLists { @Test diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDrainBarrier.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDrainBarrier.java index 4542cbd59e35..09927f294044 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDrainBarrier.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestDrainBarrier.java @@ -18,15 +18,13 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.*; + @Category({MiscTests.class, SmallTests.class}) public class TestDrainBarrier { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java index 3c7a8dd1716a..ee087ef4f083 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestEnvironmentEdgeManager.java @@ -18,19 +18,20 @@ */ package org.apache.hadoop.hbase.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + @Category({MiscTests.class, MediumTests.class}) public class TestEnvironmentEdgeManager { diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java index f5848f3d760f..a6078ab33b8d 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java @@ -20,8 +20,8 @@ import java.util.Arrays; import java.util.Random; -import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestKeyLocker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestKeyLocker.java index 9bb8a0481927..bb0590088d25 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestKeyLocker.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestKeyLocker.java @@ -18,14 +18,14 @@ package org.apache.hadoop.hbase.util; -import java.util.concurrent.locks.ReentrantLock; - import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.util.concurrent.locks.ReentrantLock; + @Category({MiscTests.class, SmallTests.class}) public class TestKeyLocker { @Test diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestShowProperties.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestShowProperties.java index 2f16ee8f7fa7..e6d154d31d44 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestShowProperties.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestShowProperties.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hbase.util; -import java.util.Properties; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -27,6 +25,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import java.util.Properties; + /** * This test is there to dump the properties. It allows to detect possible env issues when diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java index a628e98ee919..e8a0d055ac9e 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java @@ -20,8 +20,6 @@ import static org.junit.Assert.assertTrue; -import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -29,6 +27,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import java.util.concurrent.atomic.AtomicBoolean; + @Category({MiscTests.class, SmallTests.class}) public class TestThreads { private static final Log LOG = LogFactory.getLog(TestThreads.class); diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml index 781f5befba1e..52c3163f5a89 100644 --- a/hbase-examples/pom.xml +++ b/hbase-examples/pom.xml @@ -31,32 +31,13 @@ Examples of HBase usage - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - + + org.apache.maven.plugins + maven-site-plugin + + true + + maven-assembly-plugin diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml index a7ad0abf1875..ce59c6845c3f 100644 --- a/hbase-hadoop-compat/pom.xml +++ b/hbase-hadoop-compat/pom.xml @@ -35,26 +35,7 @@ - - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - + org.apache.maven.plugins maven-site-plugin diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactory.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactory.java index d6b13924dca1..4ad9f335a275 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactory.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactory.java @@ -47,9 +47,9 @@ public abstract class MetricsHBaseServerSourceFactory { * @return The Camel Cased context name. */ protected static String createContextName(String serverName) { - if (serverName.contains("HMaster") || serverName.contains("master")) { + if (serverName.contains("HMaster")) { return "Master"; - } else if (serverName.contains("HRegion") || serverName.contains("regionserver")) { + } else if (serverName.contains("HRegion")) { return "RegionServer"; } return "IPC"; diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index e845520399be..ec0ab21e0cd5 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -34,37 +34,18 @@ limitations under the License. - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - org.apache.maven.plugins - maven-source-plugin - + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + maven-assembly-plugin diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.java index cca53e0b7291..df40e682c026 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.java @@ -46,7 +46,7 @@ private static synchronized MetricsHBaseServerSource getSource(String serverName if (source == null) { //Create the source. source = new MetricsHBaseServerSourceImpl( - context, + METRICS_NAME, METRICS_DESCRIPTION, context.toLowerCase(), context + METRICS_JMX_CONTEXT_SUFFIX, wrapper); diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 96aedc969262..f4b6b2efea06 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -117,25 +117,6 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - org.apache.maven.plugins diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java index f45fb048314f..085c252b7195 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java @@ -178,9 +178,6 @@ protected IntegrationTestingUtility getTestingUtil(Configuration conf) { public abstract void setUpCluster() throws Exception; public void cleanUpCluster() throws Exception { - if (util.isDistributedCluster() && (monkey == null || !monkey.isDestructive())) { - noClusterCleanUp = true; - } if (noClusterCleanUp) { LOG.debug("noClusterCleanUp is set, skip restoring the cluster"); return; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java index 3c29f4cc958b..cc22e43cb818 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.java @@ -19,13 +19,13 @@ package org.apache.hadoop.hbase; import com.google.common.base.Objects; -import com.yammer.metrics.core.Histogram; import org.apache.commons.cli.CommandLine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.RestartRandomRsExceptMetaAction; +import org.apache.hadoop.hbase.chaos.actions.RestartRsHoldingTableAction; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; import org.apache.hadoop.hbase.chaos.policies.Policy; @@ -33,18 +33,20 @@ import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.testclassification.IntegrationTests; -import org.apache.hadoop.hbase.util.YammerHistogramUtils; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.ToolRunner; import org.junit.experimental.categories.Category; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.Queue; +import java.util.Set; import java.util.concurrent.Callable; import static java.lang.String.format; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -64,6 +66,8 @@ public class IntegrationTestRegionReplicaPerf extends IntegrationTestBase { private static final String SLEEP_TIME_DEFAULT = "" + (10 * 1000l); private static final String TABLE_NAME_KEY = "tableName"; private static final String TABLE_NAME_DEFAULT = "IntegrationTestRegionReplicaPerf"; + private static final String NOMAPRED_KEY = "nomapred"; + private static final boolean NOMAPRED_DEFAULT = false; private static final String REPLICA_COUNT_KEY = "replicas"; private static final String REPLICA_COUNT_DEFAULT = "" + 3; private static final String PRIMARY_TIMEOUT_KEY = "timeout"; @@ -71,26 +75,9 @@ public class IntegrationTestRegionReplicaPerf extends IntegrationTestBase { private static final String NUM_RS_KEY = "numRs"; private static final String NUM_RS_DEFAULT = "" + 3; - /** Extract a descriptive statistic from a {@link com.yammer.metrics.core.Histogram}. */ - private enum Stat { - STDEV { - @Override - double apply(Histogram hist) { - return hist.stdDev(); - } - }, - FOUR_9S { - @Override - double apply(Histogram hist) { - return hist.getSnapshot().getValue(0.9999); - } - }; - - abstract double apply(Histogram hist); - } - private TableName tableName; private long sleepTime; + private boolean nomapred = NOMAPRED_DEFAULT; private int replicaCount; private int primaryTimeout; private int clusterSize; @@ -113,21 +100,17 @@ public PerfEvalCallable(Admin admin, String argv) { public TimingResult call() throws Exception { PerformanceEvaluation.TestOptions opts = PerformanceEvaluation.parseOpts(argv); PerformanceEvaluation.checkTable(admin, opts); - PerformanceEvaluation.RunResult results[] = null; long numRows = opts.totalRows; - long elapsedTime = 0; + long elapsedTime; if (opts.nomapred) { - results = PerformanceEvaluation.doLocalClients(opts, admin.getConfiguration()); - for (PerformanceEvaluation.RunResult r : results) { - elapsedTime = Math.max(elapsedTime, r.duration); - } + elapsedTime = PerformanceEvaluation.doLocalClients(opts, admin.getConfiguration()); } else { Job job = PerformanceEvaluation.doMapReduce(opts, admin.getConfiguration()); Counters counters = job.getCounters(); numRows = counters.findCounter(PerformanceEvaluation.Counter.ROWS).getValue(); elapsedTime = counters.findCounter(PerformanceEvaluation.Counter.ELAPSED_TIME).getValue(); } - return new TimingResult(numRows, elapsedTime, results); + return new TimingResult(numRows, elapsedTime); } } @@ -135,14 +118,12 @@ public TimingResult call() throws Exception { * Record the results from a single {@link PerformanceEvaluation} job run. */ static class TimingResult { - public final long numRows; - public final long elapsedTime; - public final PerformanceEvaluation.RunResult results[]; + public long numRows; + public long elapsedTime; - public TimingResult(long numRows, long elapsedTime, PerformanceEvaluation.RunResult results[]) { + public TimingResult(long numRows, long elapsedTime) { this.numRows = numRows; this.elapsedTime = elapsedTime; - this.results = results; } @Override @@ -184,7 +165,7 @@ public void setUpCluster() throws Exception { @Override public void setUpMonkey() throws Exception { Policy p = new PeriodicRandomActionPolicy(sleepTime, - new RestartRandomRsExceptMetaAction(sleepTime), + new RestartRsHoldingTableAction(sleepTime, tableName.getNameAsString()), new MoveRandomRegionOfTableAction(tableName)); this.monkey = new PolicyBasedChaosMonkey(util, p); // don't start monkey right away @@ -196,18 +177,21 @@ protected void addOptions() { + TABLE_NAME_DEFAULT + "'"); addOptWithArg(SLEEP_TIME_KEY, "How long the monkey sleeps between actions. Default: " + SLEEP_TIME_DEFAULT); + addOptNoArg(NOMAPRED_KEY, + "Run multiple clients using threads (rather than use mapreduce)"); addOptWithArg(REPLICA_COUNT_KEY, "Number of region replicas. Default: " + REPLICA_COUNT_DEFAULT); addOptWithArg(PRIMARY_TIMEOUT_KEY, "Overrides hbase.client.primaryCallTimeout. Default: " + PRIMARY_TIMEOUT_DEFAULT + " (10ms)"); addOptWithArg(NUM_RS_KEY, "Specify the number of RegionServers to use. Default: " - + NUM_RS_DEFAULT); + + NUM_RS_DEFAULT); } @Override protected void processOptions(CommandLine cmd) { tableName = TableName.valueOf(cmd.getOptionValue(TABLE_NAME_KEY, TABLE_NAME_DEFAULT)); sleepTime = Long.parseLong(cmd.getOptionValue(SLEEP_TIME_KEY, SLEEP_TIME_DEFAULT)); + nomapred = cmd.hasOption(NOMAPRED_KEY); replicaCount = Integer.parseInt(cmd.getOptionValue(REPLICA_COUNT_KEY, REPLICA_COUNT_DEFAULT)); primaryTimeout = Integer.parseInt(cmd.getOptionValue(PRIMARY_TIMEOUT_KEY, PRIMARY_TIMEOUT_DEFAULT)); @@ -215,6 +199,7 @@ protected void processOptions(CommandLine cmd) { LOG.debug(Objects.toStringHelper("Parsed Options") .add(TABLE_NAME_KEY, tableName) .add(SLEEP_TIME_KEY, sleepTime) + .add(NOMAPRED_KEY, nomapred) .add(REPLICA_COUNT_KEY, replicaCount) .add(PRIMARY_TIMEOUT_KEY, primaryTimeout) .add(NUM_RS_KEY, clusterSize) @@ -237,35 +222,20 @@ protected Set getColumnFamilies() { return null; } - /** Compute the mean of the given {@code stat} from a timing results. */ - private static double calcMean(String desc, Stat stat, List results) { - double sum = 0; - int count = 0; - - for (TimingResult tr : results) { - for (PerformanceEvaluation.RunResult r : tr.results) { - assertNotNull("One of the run results is missing detailed run data.", r.hist); - sum += stat.apply(r.hist); - count += 1; - LOG.debug(desc + "{" + YammerHistogramUtils.getHistogramReport(r.hist) + "}"); - } - } - return sum / count; - } - public void test() throws Exception { int maxIters = 3; + String mr = nomapred ? "--nomapred" : ""; String replicas = "--replicas=" + replicaCount; // TODO: splits disabled until "phase 2" is complete. String splitPolicy = "--splitPolicy=" + DisabledRegionSplitPolicy.class.getName(); - String writeOpts = format("%s --nomapred --table=%s --presplit=16 sequentialWrite 4", - splitPolicy, tableName); + String writeOpts = format("%s %s --table=%s --presplit=16 sequentialWrite 4", + mr, splitPolicy, tableName); String readOpts = - format("--nomapred --table=%s --latency --sampleRate=0.1 randomRead 4", tableName); + format("%s --table=%s --latency --sampleRate=0.1 randomRead 4", mr, tableName); String replicaReadOpts = format("%s %s", replicas, readOpts); - ArrayList resultsWithoutReplicas = new ArrayList(maxIters); - ArrayList resultsWithReplicas = new ArrayList(maxIters); + ArrayList resultsWithoutReplica = new ArrayList(maxIters); + ArrayList resultsWithReplica = new ArrayList(maxIters); // create/populate the table, replicas disabled LOG.debug("Populating table."); @@ -273,14 +243,14 @@ public void test() throws Exception { // one last sanity check, then send in the clowns! assertEquals("Table must be created with DisabledRegionSplitPolicy. Broken test.", - DisabledRegionSplitPolicy.class.getName(), - util.getHBaseAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName()); + DisabledRegionSplitPolicy.class.getName(), + util.getHBaseAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName()); startMonkey(); // collect a baseline without region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching non-replica job " + (i + 1) + "/" + maxIters); - resultsWithoutReplicas.add(new PerfEvalCallable(util.getHBaseAdmin(), readOpts).call()); + resultsWithoutReplica.add(new PerfEvalCallable(util.getHBaseAdmin(), readOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } @@ -288,48 +258,39 @@ public void test() throws Exception { // disable monkey, enable region replicas, enable monkey cleanUpMonkey("Altering table."); LOG.debug("Altering " + tableName + " replica count to " + replicaCount); - IntegrationTestingUtility.setReplicas(util.getHBaseAdmin(), tableName, replicaCount); + util.setReplicas(util.getHBaseAdmin(), tableName, replicaCount); setUpMonkey(); startMonkey(); // run test with region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching replica job " + (i + 1) + "/" + maxIters); - resultsWithReplicas.add(new PerfEvalCallable(util.getHBaseAdmin(), replicaReadOpts).call()); + resultsWithReplica.add(new PerfEvalCallable(util.getHBaseAdmin(), replicaReadOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } - // compare the average of the stdev and 99.99pct across runs to determine if region replicas - // are having an overall improvement on response variance experienced by clients. - double withoutReplicasStdevMean = - calcMean("withoutReplicas", Stat.STDEV, resultsWithoutReplicas); - double withoutReplicas9999Mean = - calcMean("withoutReplicas", Stat.FOUR_9S, resultsWithoutReplicas); - double withReplicasStdevMean = - calcMean("withReplicas", Stat.STDEV, resultsWithReplicas); - double withReplicas9999Mean = - calcMean("withReplicas", Stat.FOUR_9S, resultsWithReplicas); - - LOG.info(Objects.toStringHelper(this) - .add("withoutReplicas", resultsWithoutReplicas) - .add("withReplicas", resultsWithReplicas) - .add("withoutReplicasStdevMean", withoutReplicasStdevMean) - .add("withoutReplicas99.99Mean", withoutReplicas9999Mean) - .add("withReplicasStdevMean", withReplicasStdevMean) - .add("withReplicas99.99Mean", withReplicas9999Mean) + DescriptiveStatistics withoutReplicaStats = new DescriptiveStatistics(); + for (TimingResult tr : resultsWithoutReplica) { + withoutReplicaStats.addValue(tr.elapsedTime); + } + DescriptiveStatistics withReplicaStats = new DescriptiveStatistics(); + for (TimingResult tr : resultsWithReplica) { + withReplicaStats.addValue(tr.elapsedTime); + } + + LOG.info(Objects.toStringHelper("testName") + .add("withoutReplicas", resultsWithoutReplica) + .add("withReplicas", resultsWithReplica) + .add("withoutReplicasMean", withoutReplicaStats.getMean()) + .add("withReplicasMean", withReplicaStats.getMean()) .toString()); assertTrue( - "Running with region replicas under chaos should have less request variance than without. " - + "withReplicas.stdev.mean: " + withReplicasStdevMean + "ms " - + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms.", - withReplicasStdevMean <= withoutReplicasStdevMean); - assertTrue( - "Running with region replicas under chaos should improve 99.99pct latency. " - + "withReplicas.99.99.mean: " + withReplicas9999Mean + "ms " - + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms.", - withReplicas9999Mean <= withoutReplicas9999Mean); + "Running with region replicas under chaos should be as fast or faster than without. " + + "withReplicas.mean: " + withReplicaStats.getMean() + "ms " + + "withoutReplicas.mean: " + withoutReplicaStats.getMean() + "ms.", + withReplicaStats.getMean() <= withoutReplicaStats.getMean()); } public static void main(String[] args) throws Exception { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java index c01ce0fd30c0..dea412f49a54 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java @@ -43,43 +43,19 @@ */ public class Action { - public static final String KILL_MASTER_TIMEOUT_KEY = - "hbase.chaosmonkey.action.killmastertimeout"; - public static final String START_MASTER_TIMEOUT_KEY = - "hbase.chaosmonkey.action.startmastertimeout"; - public static final String KILL_RS_TIMEOUT_KEY = "hbase.chaosmonkey.action.killrstimeout"; - public static final String START_RS_TIMEOUT_KEY = "hbase.chaosmonkey.action.startrstimeout"; - protected static Log LOG = LogFactory.getLog(Action.class); - protected static final long KILL_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; - protected static final long START_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; - protected static final long KILL_RS_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; - protected static final long START_RS_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; - protected ActionContext context; protected HBaseCluster cluster; protected ClusterStatus initialStatus; protected ServerName[] initialServers; - protected long killMasterTimeout; - protected long startMasterTimeout; - protected long killRsTimeout; - protected long startRsTimeout; - public void init(ActionContext context) throws IOException { this.context = context; cluster = context.getHBaseCluster(); initialStatus = cluster.getInitialClusterStatus(); Collection regionServers = initialStatus.getServers(); initialServers = regionServers.toArray(new ServerName[regionServers.size()]); - - killMasterTimeout = cluster.getConf().getLong(KILL_MASTER_TIMEOUT_KEY, - KILL_MASTER_TIMEOUT_DEFAULT); - startMasterTimeout = cluster.getConf().getLong(START_MASTER_TIMEOUT_KEY, - START_MASTER_TIMEOUT_DEFAULT); - killRsTimeout = cluster.getConf().getLong(KILL_RS_TIMEOUT_KEY, KILL_RS_TIMEOUT_DEFAULT); - startRsTimeout = cluster.getConf().getLong(START_RS_TIMEOUT_KEY, START_RS_TIMEOUT_DEFAULT); } public void perform() throws Exception { } @@ -108,21 +84,21 @@ protected ServerName[] getCurrentServers() throws IOException { protected void killMaster(ServerName server) throws IOException { LOG.info("Killing master:" + server); cluster.killMaster(server); - cluster.waitForMasterToStop(server, killMasterTimeout); + cluster.waitForMasterToStop(server, PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Killed master server:" + server); } protected void startMaster(ServerName server) throws IOException { LOG.info("Starting master:" + server.getHostname()); cluster.startMaster(server.getHostname()); - cluster.waitForActiveAndReadyMaster(startMasterTimeout); + cluster.waitForActiveAndReadyMaster(PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Started master: " + server); } protected void killRs(ServerName server) throws IOException { LOG.info("Killing region server:" + server); cluster.killRegionServer(server); - cluster.waitForRegionServerToStop(server, killRsTimeout); + cluster.waitForRegionServerToStop(server, PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Killed region server:" + server + ". Reported num of rs:" + cluster.getClusterStatus().getServersSize()); } @@ -130,7 +106,7 @@ protected void killRs(ServerName server) throws IOException { protected void startRs(ServerName server) throws IOException { LOG.info("Starting region server:" + server.getHostname()); cluster.startRegionServer(server.getHostname()); - cluster.waitForRegionServerToStart(server.getHostname(), startRsTimeout); + cluster.waitForRegionServerToStart(server.getHostname(), PolicyBasedChaosMonkey.TIMEOUT); LOG.info("Started region server:" + server + ". Reported num of rs:" + cluster.getClusterStatus().getServersSize()); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java index f4b1c5306db8..bdf05d127556 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyFactory.java @@ -68,7 +68,6 @@ public MonkeyFactory setProperties(Properties props) { public static final String UNBALANCE = "unbalance"; public static final String SERVER_KILLING = "serverKilling"; public static final String STRESS_AM = "stressAM"; - public static final String NO_KILL = "noKill"; public static Map FACTORIES = ImmutableMap.builder() .put(CALM, new CalmMonkeyFactory()) @@ -76,7 +75,6 @@ public MonkeyFactory setProperties(Properties props) { .put(UNBALANCE, new UnbalanceMonkeyFactory()) .put(SERVER_KILLING, new ServerKillingMonkeyFactory()) .put(STRESS_AM, new StressAssignmentManagerMonkeyFactory()) - .put(NO_KILL, new NoKillMonkeyFactory()) .build(); public static MonkeyFactory getFactory(String factoryName) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java deleted file mode 100644 index def9df0bfa80..000000000000 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.chaos.factories; - -import org.apache.hadoop.hbase.chaos.actions.Action; -import org.apache.hadoop.hbase.chaos.actions.AddColumnAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeBloomFilterAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeCompressionAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeEncodingAction; -import org.apache.hadoop.hbase.chaos.actions.ChangeVersionsAction; -import org.apache.hadoop.hbase.chaos.actions.CompactRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.CompactTableAction; -import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction; -import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; -import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; -import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction; -import org.apache.hadoop.hbase.chaos.actions.SnapshotTableAction; -import org.apache.hadoop.hbase.chaos.actions.SplitRandomRegionOfTableAction; -import org.apache.hadoop.hbase.chaos.monkies.ChaosMonkey; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; -import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; -import org.apache.hadoop.hbase.chaos.policies.TwoConcurrentActionPolicy; - -/** - * Monkey factory to create a ChaosMonkey that will not need access to ssh. It will not - * kill any services and it will not perform any restarts. - */ -public class NoKillMonkeyFactory extends MonkeyFactory { - @Override public ChaosMonkey build() { - Action[] actions1 = new Action[] { - new CompactTableAction(tableName, MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD), - new CompactRandomRegionOfTableAction(tableName, - MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO), - new FlushTableAction(tableName), - new FlushRandomRegionOfTableAction(tableName), - new MoveRandomRegionOfTableAction(tableName) - }; - - Action[] actions2 = new Action[] { - new SplitRandomRegionOfTableAction(tableName), - new MergeRandomAdjacentRegionsOfTableAction(tableName), - new SnapshotTableAction(tableName), - new AddColumnAction(tableName), - new RemoveColumnAction(tableName, columnFamilies), - new ChangeEncodingAction(tableName), - new ChangeCompressionAction(tableName), - new ChangeBloomFilterAction(tableName), - new ChangeVersionsAction(tableName) - }; - - Action[] actions3 = new Action[] { - new MoveRegionsOfTableAction(MonkeyConstants.DEFAULT_MOVE_REGIONS_SLEEP_TIME, - MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME, - tableName), - new MoveRandomRegionOfTableAction(MonkeyConstants.DEFAULT_RESTART_ACTIVE_MASTER_SLEEP_TIME, - tableName), - }; - - Action[] actions4 = new Action[] { - new DumpClusterStatusAction() - }; - - return new PolicyBasedChaosMonkey(util, - new TwoConcurrentActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION1_PERIOD, actions1, actions2), - new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION2_PERIOD,actions3), - new PeriodicRandomActionPolicy(MonkeyConstants.DEFAULT_PERIODIC_ACTION4_PERIOD,actions4)); - } -} diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java index 619573717ced..70d64b9aa368 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.chaos.policies.CompositeSequentialPolicy; import org.apache.hadoop.hbase.chaos.policies.DoActionsOncePolicy; import org.apache.hadoop.hbase.chaos.policies.PeriodicRandomActionPolicy; +import org.junit.Assert; public class SlowDeterministicMonkeyFactory extends MonkeyFactory { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/CalmChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/CalmChaosMonkey.java index bcf7d85ee75a..bcf45dd1a23f 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/CalmChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/CalmChaosMonkey.java @@ -41,9 +41,4 @@ public boolean isStopped() { public void waitForStop() throws InterruptedException { } - - @Override - public boolean isDestructive() { - return false; - } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java index 86cbfe5c612d..da75c3be4801 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/ChaosMonkey.java @@ -51,10 +51,4 @@ public abstract class ChaosMonkey implements Stoppable { public abstract boolean isStopped(); public abstract void waitForStop() throws InterruptedException; - - /** - * Returns whether the CM does destructive actions (killing servers) so that a cluster restore - * is needed after CM is stopped. Otherwise cluster will be left as it is - */ - public abstract boolean isDestructive(); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index 9152cb2a3a09..f42f903473ea 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -149,10 +149,4 @@ public void waitForStop() throws InterruptedException { monkeyThread.join(); } } - - @Override - public boolean isDestructive() { - // TODO: we can look at the actions, and decide to do the restore cluster or not based on them. - return true; - } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/TwoConcurrentActionPolicy.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/TwoConcurrentActionPolicy.java deleted file mode 100644 index 5c45d9c74716..000000000000 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/policies/TwoConcurrentActionPolicy.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.chaos.policies; - -import org.apache.hadoop.hbase.DaemonThreadFactory; -import org.apache.hadoop.hbase.chaos.actions.Action; -import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; -import org.apache.hadoop.util.StringUtils; - -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -/** - * Chaos Monkey policy that will run two different actions at the same time. - * A random action from each array of actions will be chosen and then run in parallel. - */ -public class TwoConcurrentActionPolicy extends PeriodicPolicy { - private final Action[] actionsOne; - private final Action[] actionsTwo; - private final ExecutorService executor; - - public TwoConcurrentActionPolicy(long sleepTime, Action[] actionsOne, Action[] actionsTwo) { - super(sleepTime); - this.actionsOne = actionsOne; - this.actionsTwo = actionsTwo; - executor = Executors.newFixedThreadPool(2, - new DaemonThreadFactory("TwoConcurrentAction-")); - } - - @Override - protected void runOneIteration() { - Action actionOne = PolicyBasedChaosMonkey.selectRandomItem(actionsOne); - Action actionTwo = PolicyBasedChaosMonkey.selectRandomItem(actionsTwo); - - Future fOne = executor.submit(new ActionRunner(actionOne)); - Future fTwo = executor.submit(new ActionRunner(actionTwo)); - - try { - fOne.get(); - fTwo.get(); - } catch (InterruptedException e) { - LOG.warn("Exception occurred during performing action: " - + StringUtils.stringifyException(e)); - } catch (ExecutionException ex) { - LOG.warn("Exception occurred during performing action: " - + StringUtils.stringifyException(ex)); - } - } - - @Override - public void init(PolicyContext context) throws Exception { - super.init(context); - for (Action a : actionsOne) { - a.init(context); - } - for (Action a : actionsTwo) { - a.init(context); - } - } - - private static class ActionRunner implements Runnable { - - private final Action action; - - public ActionRunner(Action action) { - - this.action = action; - } - - @Override public void run() { - try { - action.perform(); - } catch (Exception ex) { - LOG.warn("Exception occurred during performing action: " - + StringUtils.stringifyException(ex)); - } - } - } -} diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java index 12adc80dc045..f695db9bf705 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mttr/IntegrationTestMTTR.java @@ -183,10 +183,6 @@ public static void setUp() throws Exception { } private static void setupActions() throws IOException { - // allow a little more time for RS restart actions because RS start depends on having a master - // to report to and the master is also being monkeyed. - util.getConfiguration().setLong(Action.START_RS_TIMEOUT_KEY, 3 * 60 * 1000); - // Set up the action that will restart a region server holding a region from our table // because this table should only have one region we should be good. restartRSAction = new RestartRsHoldingTableAction(sleepTime, tableName.getNameAsString()); @@ -282,7 +278,6 @@ public void testRestartMaster() throws Exception { public void run(Callable monkeyCallable, String testName) throws Exception { int maxIters = util.getHBaseClusterInterface().isDistributedCluster() ? 10 : 3; - LOG.info("Starting " + testName + " with " + maxIters + " iterations."); // Array to keep track of times. ArrayList resultPuts = new ArrayList(maxIters); @@ -290,39 +285,33 @@ public void run(Callable monkeyCallable, String testName) throws Except ArrayList resultAdmin = new ArrayList(maxIters); long start = System.nanoTime(); - try { - // We're going to try this multiple times - for (int fullIterations = 0; fullIterations < maxIters; fullIterations++) { - // Create and start executing a callable that will kill the servers - Future monkeyFuture = executorService.submit(monkeyCallable); + // We're going to try this multiple times + for (int fullIterations = 0; fullIterations < maxIters; fullIterations++) { + // Create and start executing a callable that will kill the servers + Future monkeyFuture = executorService.submit(monkeyCallable); - // Pass that future to the timing Callables. - Future putFuture = executorService.submit(new PutCallable(monkeyFuture)); - Future scanFuture = executorService.submit(new ScanCallable(monkeyFuture)); - Future adminFuture = executorService.submit(new AdminCallable(monkeyFuture)); + // Pass that future to the timing Callables. + Future putFuture = executorService.submit(new PutCallable(monkeyFuture)); + Future scanFuture = executorService.submit(new ScanCallable(monkeyFuture)); + Future adminFuture = executorService.submit(new AdminCallable(monkeyFuture)); - Future loadFuture = executorService.submit(new LoadCallable(monkeyFuture)); + Future loadFuture = executorService.submit(new LoadCallable(monkeyFuture)); - monkeyFuture.get(); - loadFuture.get(); + monkeyFuture.get(); + loadFuture.get(); - // Get the values from the futures. - TimingResult putTime = putFuture.get(); - TimingResult scanTime = scanFuture.get(); - TimingResult adminTime = adminFuture.get(); + // Get the values from the futures. + TimingResult putTime = putFuture.get(); + TimingResult scanTime = scanFuture.get(); + TimingResult adminTime = adminFuture.get(); - // Store the times to display later. - resultPuts.add(putTime); - resultScan.add(scanTime); - resultAdmin.add(adminTime); + // Store the times to display later. + resultPuts.add(putTime); + resultScan.add(scanTime); + resultAdmin.add(adminTime); - // Wait some time for everything to settle down. - Thread.sleep(5000l); - } - } catch (Exception e) { - long runtimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS); - LOG.info(testName + " failed after " + runtimeMs + "ms.", e); - throw e; + // Wait some time for everything to settle down. + Thread.sleep(5000l); } long runtimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 177341f4721a..f5e5506675af 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -39,7 +39,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -48,7 +47,6 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestBase; import org.apache.hadoop.hbase.IntegrationTestingUtility; -import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.TableName; @@ -1039,34 +1037,6 @@ private CINode getNode(byte[] row, Table table, CINode node) throws IOException } } - private static class Clean extends Configured implements Tool { - - @Override public int run(String[] args) throws Exception { - if (args.length < 1) { - System.err.println("Usage: Clean "); - return -1; - } - - Path p = new Path(args[0]); - Configuration conf = getConf(); - TableName tableName = getTableName(conf); - - FileSystem fs = HFileSystem.get(conf); - Admin admin = new HBaseAdmin(conf); - - if (admin.tableExists(tableName)) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } - - if (fs.exists(p)) { - fs.delete(p, true); - } - - return 0; - } - } - static TableName getTableName(Configuration conf) { return TableName.valueOf(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME)); } @@ -1140,7 +1110,6 @@ private void usage() { System.err.println(" single node."); System.err.println(" Loop A program to Loop through Generator and"); System.err.println(" Verify steps"); - System.err.println(" Clean A program to clean all left over detritus."); System.err.println("\t "); System.err.flush(); } @@ -1176,8 +1145,6 @@ public int runTestFromCommandLine() throws Exception { tool = new Print(); } else if (toRun.equals("Delete")) { tool = new Delete(); - } else if (toRun.equals("Clean")) { - tool = new Clean(); } else { usage(); throw new RuntimeException("Unknown arg"); diff --git a/hbase-prefix-tree/pom.xml b/hbase-prefix-tree/pom.xml index 665480635f31..2c0dd365d65c 100644 --- a/hbase-prefix-tree/pom.xml +++ b/hbase-prefix-tree/pom.xml @@ -33,37 +33,18 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - org.apache.maven.plugins - maven-source-plugin - + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + maven-assembly-plugin diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index 2db0f29aa186..fb73443cc6cb 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -24,17 +24,13 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.SettableSequenceId; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory; import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher; import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition; -import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder.EncodedSeeker; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ClassSize; /** * These methods have the same definition as any implementation of the EncodedSeeker. @@ -275,10 +271,7 @@ public int compareKey(KVComparator comparator, Cell key) { * of the key part is deep copied * */ - private static class ClonedPrefixTreeCell implements Cell, SettableSequenceId, HeapSize { - private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT - + (5 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (4 * Bytes.SIZEOF_INT) - + (Bytes.SIZEOF_SHORT) + (2 * Bytes.SIZEOF_BYTE) + (5 * ClassSize.ARRAY)); + private static class ClonedPrefixTreeCell implements Cell, SettableSequenceId { private byte[] row; private short rowLength; private byte[] fam; @@ -289,7 +282,7 @@ private static class ClonedPrefixTreeCell implements Cell, SettableSequenceId, H private int valOffset; private int valLength; private byte[] tag; - private int tagsLength; + private int tagLength; private long ts; private long seqId; private byte type; @@ -309,7 +302,7 @@ public ClonedPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] f this.qualLength = qualLength; this.tag = new byte[tagLength]; System.arraycopy(tag, tagOffset, this.tag, 0, tagLength); - this.tagsLength = tagLength; + this.tagLength = tagLength; this.val = val; this.valLength = valLength; this.valOffset = valOffset; @@ -416,7 +409,7 @@ public int getTagsOffset() { @Override public int getTagsLength() { - return this.tagsLength; + return this.tagLength; } @Override @@ -445,18 +438,7 @@ public byte[] getRow() { @Override public String toString() { - String row = Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength()); - String family = Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength()); - String qualifier = Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(), - getQualifierLength()); - String timestamp = String.valueOf((getTimestamp())); - return row + "/" + family + (family != null && family.length() > 0 ? ":" : "") + qualifier - + "/" + timestamp + "/" + Type.codeToType(type); - } - - @Override - public long heapSize() { - return FIXED_OVERHEAD + rowLength + famLength + qualLength + valLength + tagsLength; + return KeyValueUtil.copyToNewKeyValue(this).toString(); } } } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java index a68f8f8d6fc2..c413aebc3a7e 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java @@ -169,7 +169,7 @@ public boolean equals(Object obj) { //trivial override to confirm intent (findbugs) return super.equals(obj); } - + @Override public int hashCode() { return super.hashCode(); @@ -370,7 +370,7 @@ protected void markAfterLast() { /***************** helper methods **************************/ protected void appendCurrentTokenToRowBuffer() { - System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, + System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, currentRowNode.getTokenLength()); rowLength += currentRowNode.getTokenLength(); } @@ -430,7 +430,7 @@ protected boolean isLastCellInRow() { protected int populateNonRowFieldsAndCompareTo(int cellNum, Cell key) { populateNonRowFields(cellNum); - return CellComparator.compare(this, key, true); + return CellComparator.compareStatic(this, key, true); } protected void populateFirstNonRowFields() { diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java index 97eed62bae9d..c29a7043ac54 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java @@ -103,7 +103,7 @@ public int hashCode() { @Override public int compareTo(Cell other) { - return CellComparator.compare(this, other, false); + return CellComparator.compareStatic(this, other, false); } @Override diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/keyvalue/TestKeyValueTool.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/keyvalue/TestKeyValueTool.java index 9e27942cf2a3..5bd44947e2f3 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/keyvalue/TestKeyValueTool.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/keyvalue/TestKeyValueTool.java @@ -43,7 +43,7 @@ public class TestKeyValueTool { @Parameters public static Collection parameters() { - return TestRowData.InMemory.getAllAsObjectArray(); + return new TestRowData.InMemory().getAllAsObjectArray(); } private TestRowData rows; diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestPrefixTreeSearcher.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestPrefixTreeSearcher.java index 55d3d221c7e4..20303faa6f15 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestPrefixTreeSearcher.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestPrefixTreeSearcher.java @@ -52,7 +52,7 @@ public class TestPrefixTreeSearcher { @Parameters public static Collection parameters() { - return TestRowData.InMemory.getAllAsObjectArray(); + return new TestRowData.InMemory().getAllAsObjectArray(); } protected TestRowData rows; diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowData.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowData.java index 4bf60e03df3d..2eb897f9d4b5 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowData.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowData.java @@ -57,7 +57,7 @@ public interface TestRowData { void individualSearcherAssertions(CellSearcher searcher); - static class InMemory { + class InMemory { /* * The following are different styles of data that the codec may encounter. Having these small diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java index 5bc44ff2dfe0..c558485f03f3 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClusterStatusProtos.java @@ -2153,24 +2153,6 @@ public interface RegionLoadOrBuilder * */ long getCompleteSequenceId(); - - // optional float data_locality = 16; - /** - * optional float data_locality = 16; - * - * 

-     ** The current data locality for region in the regionserver 
-     *
- */ - boolean hasDataLocality(); - /** - * optional float data_locality = 16; - * - * 
-     ** The current data locality for region in the regionserver 
-     *
- */ - float getDataLocality(); } /** * Protobuf type {@code RegionLoad} @@ -2306,11 +2288,6 @@ private RegionLoad( completeSequenceId_ = input.readUInt64(); break; } - case 133: { - bitField0_ |= 0x00008000; - dataLocality_ = input.readFloat(); - break; - } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -2729,30 +2706,6 @@ public long getCompleteSequenceId() { return completeSequenceId_; } - // optional float data_locality = 16; - public static final int DATA_LOCALITY_FIELD_NUMBER = 16; - private float dataLocality_; - /** - * optional float data_locality = 16; - * - * 
-     ** The current data locality for region in the regionserver 
-     *
- */ - public boolean hasDataLocality() { - return ((bitField0_ & 0x00008000) == 0x00008000); - } - /** - * optional float data_locality = 16; - * - * 
-     ** The current data locality for region in the regionserver 
-     *
- */ - public float getDataLocality() { - return dataLocality_; - } - private void initFields() { regionSpecifier_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance(); stores_ = 0; @@ -2769,7 +2722,6 @@ private void initFields() { totalStaticIndexSizeKB_ = 0; totalStaticBloomSizeKB_ = 0; completeSequenceId_ = 0L; - dataLocality_ = 0F; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2836,9 +2788,6 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00004000) == 0x00004000)) { output.writeUInt64(15, completeSequenceId_); } - if (((bitField0_ & 0x00008000) == 0x00008000)) { - output.writeFloat(16, dataLocality_); - } getUnknownFields().writeTo(output); } @@ -2908,10 +2857,6 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeUInt64Size(15, completeSequenceId_); } - if (((bitField0_ & 0x00008000) == 0x00008000)) { - size += com.google.protobuf.CodedOutputStream - .computeFloatSize(16, dataLocality_); - } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -3010,10 +2955,6 @@ public boolean equals(final java.lang.Object obj) { result = result && (getCompleteSequenceId() == other.getCompleteSequenceId()); } - result = result && (hasDataLocality() == other.hasDataLocality()); - if (hasDataLocality()) { - result = result && (Float.floatToIntBits(getDataLocality()) == Float.floatToIntBits(other.getDataLocality())); - } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -3087,11 +3028,6 @@ public int hashCode() { hash = (37 * hash) + COMPLETE_SEQUENCE_ID_FIELD_NUMBER; hash = (53 * hash) + hashLong(getCompleteSequenceId()); } - if (hasDataLocality()) { - hash = (37 * hash) + DATA_LOCALITY_FIELD_NUMBER; - hash = (53 * hash) + Float.floatToIntBits( - getDataLocality()); - } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -3236,8 +3172,6 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00002000); completeSequenceId_ = 0L; bitField0_ = (bitField0_ & ~0x00004000); - dataLocality_ = 0F; - bitField0_ = (bitField0_ & ~0x00008000); return this; } @@ -3330,10 +3264,6 @@ public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad to_bitField0_ |= 0x00004000; } result.completeSequenceId_ = completeSequenceId_; - if (((from_bitField0_ & 0x00008000) == 0x00008000)) { - to_bitField0_ |= 0x00008000; - } - result.dataLocality_ = dataLocality_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -3395,9 +3325,6 @@ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ClusterStatu if (other.hasCompleteSequenceId()) { setCompleteSequenceId(other.getCompleteSequenceId()); } - if (other.hasDataLocality()) { - setDataLocality(other.getDataLocality()); - } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -4288,55 +4215,6 @@ public Builder clearCompleteSequenceId() { return this; } - // optional float data_locality = 16; - private float dataLocality_ ; - /** - * optional float data_locality = 16; - * - * 
-       ** The current data locality for region in the regionserver 
-       *
- */ - public boolean hasDataLocality() { - return ((bitField0_ & 0x00008000) == 0x00008000); - } - /** - * optional float data_locality = 16; - * - * 
-       ** The current data locality for region in the regionserver 
-       *
- */ - public float getDataLocality() { - return dataLocality_; - } - /** - * optional float data_locality = 16; - * - * 
-       ** The current data locality for region in the regionserver 
-       *
- */ - public Builder setDataLocality(float value) { - bitField0_ |= 0x00008000; - dataLocality_ = value; - onChanged(); - return this; - } - /** - * optional float data_locality = 16; - * - * 
-       ** The current data locality for region in the regionserver 
-       *
- */ - public Builder clearDataLocality() { - bitField0_ = (bitField0_ & ~0x00008000); - dataLocality_ = 0F; - onChanged(); - return this; - } - // @@protoc_insertion_point(builder_scope:RegionLoad) } @@ -10472,7 +10350,7 @@ public Builder clearBalancerOn() { "PLITTING_NEW\020\r\022\017\n\013MERGING_NEW\020\016\"X\n\022Regio", "nInTransition\022\036\n\004spec\030\001 \002(\0132\020.RegionSpec" + "ifier\022\"\n\014region_state\030\002 \002(\0132\014.RegionStat" + - "e\"\347\003\n\nRegionLoad\022*\n\020region_specifier\030\001 \002" + + "e\"\320\003\n\nRegionLoad\022*\n\020region_specifier\030\001 \002" + "(\0132\020.RegionSpecifier\022\016\n\006stores\030\002 \001(\r\022\022\n\n" + "storefiles\030\003 \001(\r\022\"\n\032store_uncompressed_s" + "ize_MB\030\004 \001(\r\022\031\n\021storefile_size_MB\030\005 \001(\r\022" + @@ -10483,27 +10361,27 @@ public Builder clearBalancerOn() { "ompacted_KVs\030\013 \001(\004\022\032\n\022root_index_size_KB" + "\030\014 \001(\r\022\"\n\032total_static_index_size_KB\030\r \001" + "(\r\022\"\n\032total_static_bloom_size_KB\030\016 \001(\r\022\034" + - "\n\024complete_sequence_id\030\017 \001(\004\022\025\n\rdata_loc" + - "ality\030\020 \001(\002\"\212\002\n\nServerLoad\022\032\n\022number_of_" + - "requests\030\001 \001(\r\022 \n\030total_number_of_reques" + - "ts\030\002 \001(\r\022\024\n\014used_heap_MB\030\003 \001(\r\022\023\n\013max_he" + - "ap_MB\030\004 \001(\r\022!\n\014region_loads\030\005 \003(\0132\013.Regi" + - "onLoad\022\"\n\014coprocessors\030\006 \003(\0132\014.Coprocess" + - "or\022\031\n\021report_start_time\030\007 \001(\004\022\027\n\017report_", - "end_time\030\010 \001(\004\022\030\n\020info_server_port\030\t \001(\r" + - "\"O\n\016LiveServerInfo\022\033\n\006server\030\001 \002(\0132\013.Ser" + - "verName\022 \n\013server_load\030\002 \002(\0132\013.ServerLoa" + - "d\"\340\002\n\rClusterStatus\022/\n\rhbase_version\030\001 \001" + - "(\0132\030.HBaseVersionFileContent\022%\n\014live_ser" + - "vers\030\002 \003(\0132\017.LiveServerInfo\022!\n\014dead_serv" + - "ers\030\003 \003(\0132\013.ServerName\0222\n\025regions_in_tra" + - "nsition\030\004 \003(\0132\023.RegionInTransition\022\036\n\ncl" + - "uster_id\030\005 \001(\0132\n.ClusterId\022)\n\023master_cop" + - "rocessors\030\006 \003(\0132\014.Coprocessor\022\033\n\006master\030", - "\007 \001(\0132\013.ServerName\022#\n\016backup_masters\030\010 \003" + - "(\0132\013.ServerName\022\023\n\013balancer_on\030\t \001(\010BF\n*" + - "org.apache.hadoop.hbase.protobuf.generat" + - "edB\023ClusterStatusProtosH\001\240\001\001" + "\n\024complete_sequence_id\030\017 \001(\004\"\212\002\n\nServerL" + + "oad\022\032\n\022number_of_requests\030\001 \001(\r\022 \n\030total" + + "_number_of_requests\030\002 \001(\r\022\024\n\014used_heap_M" + + "B\030\003 \001(\r\022\023\n\013max_heap_MB\030\004 \001(\r\022!\n\014region_l" + + "oads\030\005 \003(\0132\013.RegionLoad\022\"\n\014coprocessors\030" + + "\006 \003(\0132\014.Coprocessor\022\031\n\021report_start_time" + + "\030\007 \001(\004\022\027\n\017report_end_time\030\010 \001(\004\022\030\n\020info_", + "server_port\030\t \001(\r\"O\n\016LiveServerInfo\022\033\n\006s" + + "erver\030\001 \002(\0132\013.ServerName\022 \n\013server_load\030" + + "\002 \002(\0132\013.ServerLoad\"\340\002\n\rClusterStatus\022/\n\r" + + "hbase_version\030\001 \001(\0132\030.HBaseVersionFileCo" + + "ntent\022%\n\014live_servers\030\002 \003(\0132\017.LiveServer" + + "Info\022!\n\014dead_servers\030\003 \003(\0132\013.ServerName\022" + + "2\n\025regions_in_transition\030\004 \003(\0132\023.RegionI" + + "nTransition\022\036\n\ncluster_id\030\005 \001(\0132\n.Cluste" + + "rId\022)\n\023master_coprocessors\030\006 \003(\0132\014.Copro" + + "cessor\022\033\n\006master\030\007 \001(\0132\013.ServerName\022#\n\016b", + "ackup_masters\030\010 \003(\0132\013.ServerName\022\023\n\013bala" + + "ncer_on\030\t \001(\010BF\n*org.apache.hadoop.hbase" + + ".protobuf.generatedB\023ClusterStatusProtos" + + "H\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -10527,7 +10405,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_RegionLoad_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_RegionLoad_descriptor, - new java.lang.String[] { "RegionSpecifier", "Stores", "Storefiles", "StoreUncompressedSizeMB", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "TotalCompactingKVs", "CurrentCompactedKVs", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "CompleteSequenceId", "DataLocality", }); + new java.lang.String[] { "RegionSpecifier", "Stores", "Storefiles", "StoreUncompressedSizeMB", "StorefileSizeMB", "MemstoreSizeMB", "StorefileIndexSizeMB", "ReadRequestsCount", "WriteRequestsCount", "TotalCompactingKVs", "CurrentCompactedKVs", "RootIndexSizeKB", "TotalStaticIndexSizeKB", "TotalStaticBloomSizeKB", "CompleteSequenceId", }); internal_static_ServerLoad_descriptor = getDescriptor().getMessageTypes().get(3); internal_static_ServerLoad_fieldAccessorTable = new diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ComparatorProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ComparatorProtos.java index ea42a81645dd..d4b850eba1ca 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ComparatorProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ComparatorProtos.java @@ -1608,559 +1608,6 @@ public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComp // @@protoc_insertion_point(class_scope:BinaryComparator) } - public interface LongComparatorOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // required .ByteArrayComparable comparable = 1; - /** - * required .ByteArrayComparable comparable = 1; - */ - boolean hasComparable(); - /** - * required .ByteArrayComparable comparable = 1; - */ - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable getComparable(); - /** - * required .ByteArrayComparable comparable = 1; - */ - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder getComparableOrBuilder(); - } - /** - * Protobuf type {@code LongComparator} - */ - public static final class LongComparator extends - com.google.protobuf.GeneratedMessage - implements LongComparatorOrBuilder { - // Use LongComparator.newBuilder() to construct. - private LongComparator(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private LongComparator(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final LongComparator defaultInstance; - public static LongComparator getDefaultInstance() { - return defaultInstance; - } - - public LongComparator getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private LongComparator( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 10: { - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder subBuilder = null; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - subBuilder = comparable_.toBuilder(); - } - comparable_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.PARSER, extensionRegistry); - if (subBuilder != null) { - subBuilder.mergeFrom(comparable_); - comparable_ = subBuilder.buildPartial(); - } - bitField0_ |= 0x00000001; - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.internal_static_LongComparator_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.internal_static_LongComparator_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.class, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public LongComparator parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new LongComparator(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // required .ByteArrayComparable comparable = 1; - public static final int COMPARABLE_FIELD_NUMBER = 1; - private org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable comparable_; - /** - * required .ByteArrayComparable comparable = 1; - */ - public boolean hasComparable() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable getComparable() { - return comparable_; - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder getComparableOrBuilder() { - return comparable_; - } - - private void initFields() { - comparable_ = org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.getDefaultInstance(); - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - if (!hasComparable()) { - memoizedIsInitialized = 0; - return false; - } - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeMessage(1, comparable_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(1, comparable_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator)) { - return super.equals(obj); - } - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator other = (org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator) obj; - - boolean result = true; - result = result && (hasComparable() == other.hasComparable()); - if (hasComparable()) { - result = result && getComparable() - .equals(other.getComparable()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasComparable()) { - hash = (37 * hash) + COMPARABLE_FIELD_NUMBER; - hash = (53 * hash) + getComparable().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code LongComparator} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparatorOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.internal_static_LongComparator_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.internal_static_LongComparator_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.class, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.Builder.class); - } - - // Construct using org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - getComparableFieldBuilder(); - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - if (comparableBuilder_ == null) { - comparable_ = org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.getDefaultInstance(); - } else { - comparableBuilder_.clear(); - } - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.internal_static_LongComparator_descriptor; - } - - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator getDefaultInstanceForType() { - return org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.getDefaultInstance(); - } - - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator build() { - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator buildPartial() { - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator result = new org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - if (comparableBuilder_ == null) { - result.comparable_ = comparable_; - } else { - result.comparable_ = comparableBuilder_.build(); - } - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator) { - return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator other) { - if (other == org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator.getDefaultInstance()) return this; - if (other.hasComparable()) { - mergeComparable(other.getComparable()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - if (!hasComparable()) { - - return false; - } - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.LongComparator) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // required .ByteArrayComparable comparable = 1; - private org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable comparable_ = org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.getDefaultInstance(); - private com.google.protobuf.SingleFieldBuilder< - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder> comparableBuilder_; - /** - * required .ByteArrayComparable comparable = 1; - */ - public boolean hasComparable() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable getComparable() { - if (comparableBuilder_ == null) { - return comparable_; - } else { - return comparableBuilder_.getMessage(); - } - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public Builder setComparable(org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable value) { - if (comparableBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - comparable_ = value; - onChanged(); - } else { - comparableBuilder_.setMessage(value); - } - bitField0_ |= 0x00000001; - return this; - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public Builder setComparable( - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder builderForValue) { - if (comparableBuilder_ == null) { - comparable_ = builderForValue.build(); - onChanged(); - } else { - comparableBuilder_.setMessage(builderForValue.build()); - } - bitField0_ |= 0x00000001; - return this; - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public Builder mergeComparable(org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable value) { - if (comparableBuilder_ == null) { - if (((bitField0_ & 0x00000001) == 0x00000001) && - comparable_ != org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.getDefaultInstance()) { - comparable_ = - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.newBuilder(comparable_).mergeFrom(value).buildPartial(); - } else { - comparable_ = value; - } - onChanged(); - } else { - comparableBuilder_.mergeFrom(value); - } - bitField0_ |= 0x00000001; - return this; - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public Builder clearComparable() { - if (comparableBuilder_ == null) { - comparable_ = org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.getDefaultInstance(); - onChanged(); - } else { - comparableBuilder_.clear(); - } - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder getComparableBuilder() { - bitField0_ |= 0x00000001; - onChanged(); - return getComparableFieldBuilder().getBuilder(); - } - /** - * required .ByteArrayComparable comparable = 1; - */ - public org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder getComparableOrBuilder() { - if (comparableBuilder_ != null) { - return comparableBuilder_.getMessageOrBuilder(); - } else { - return comparable_; - } - } - /** - * required .ByteArrayComparable comparable = 1; - */ - private com.google.protobuf.SingleFieldBuilder< - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder> - getComparableFieldBuilder() { - if (comparableBuilder_ == null) { - comparableBuilder_ = new com.google.protobuf.SingleFieldBuilder< - org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparable.Builder, org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos.ByteArrayComparableOrBuilder>( - comparable_, - getParentForChildren(), - isClean()); - comparable_ = null; - } - return comparableBuilder_; - } - - // @@protoc_insertion_point(builder_scope:LongComparator) - } - - static { - defaultInstance = new LongComparator(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:LongComparator) - } - public interface BinaryPrefixComparatorOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -5288,11 +4735,6 @@ public Builder setSubstrBytes( private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_BinaryComparator_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_LongComparator_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_LongComparator_fieldAccessorTable; private static com.google.protobuf.Descriptors.Descriptor internal_static_BinaryPrefixComparator_descriptor; private static @@ -5331,19 +4773,18 @@ public Builder setSubstrBytes( "\030\001 \002(\t\022\035\n\025serialized_comparator\030\002 \001(\014\"$\n" + "\023ByteArrayComparable\022\r\n\005value\030\001 \001(\014\"<\n\020B" + "inaryComparator\022(\n\ncomparable\030\001 \002(\0132\024.By" + - "teArrayComparable\":\n\016LongComparator\022(\n\nc" + - "omparable\030\001 \002(\0132\024.ByteArrayComparable\"B\n" + - "\026BinaryPrefixComparator\022(\n\ncomparable\030\001 " + - "\002(\0132\024.ByteArrayComparable\"\216\001\n\rBitCompara" + + "teArrayComparable\"B\n\026BinaryPrefixCompara" + "tor\022(\n\ncomparable\030\001 \002(\0132\024.ByteArrayCompa" + - "rable\022,\n\nbitwise_op\030\002 \002(\0162\030.BitComparato", - "r.BitwiseOp\"%\n\tBitwiseOp\022\007\n\003AND\020\001\022\006\n\002OR\020" + - "\002\022\007\n\003XOR\020\003\"\020\n\016NullComparator\"`\n\025RegexStr" + - "ingComparator\022\017\n\007pattern\030\001 \002(\t\022\025\n\rpatter" + - "n_flags\030\002 \002(\005\022\017\n\007charset\030\003 \002(\t\022\016\n\006engine" + - "\030\004 \001(\t\"%\n\023SubstringComparator\022\016\n\006substr\030" + - "\001 \002(\tBF\n*org.apache.hadoop.hbase.protobu" + - "f.generatedB\020ComparatorProtosH\001\210\001\001\240\001\001" + "rable\"\216\001\n\rBitComparator\022(\n\ncomparable\030\001 " + + "\002(\0132\024.ByteArrayComparable\022,\n\nbitwise_op\030" + + "\002 \002(\0162\030.BitComparator.BitwiseOp\"%\n\tBitwi" + + "seOp\022\007\n\003AND\020\001\022\006\n\002OR\020\002\022\007\n\003XOR\020\003\"\020\n\016NullCo", + "mparator\"`\n\025RegexStringComparator\022\017\n\007pat" + + "tern\030\001 \002(\t\022\025\n\rpattern_flags\030\002 \002(\005\022\017\n\007cha" + + "rset\030\003 \002(\t\022\016\n\006engine\030\004 \001(\t\"%\n\023SubstringC" + + "omparator\022\016\n\006substr\030\001 \002(\tBF\n*org.apache." + + "hadoop.hbase.protobuf.generatedB\020Compara" + + "torProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -5368,38 +4809,32 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_BinaryComparator_descriptor, new java.lang.String[] { "Comparable", }); - internal_static_LongComparator_descriptor = - getDescriptor().getMessageTypes().get(3); - internal_static_LongComparator_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_LongComparator_descriptor, - new java.lang.String[] { "Comparable", }); internal_static_BinaryPrefixComparator_descriptor = - getDescriptor().getMessageTypes().get(4); + getDescriptor().getMessageTypes().get(3); internal_static_BinaryPrefixComparator_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_BinaryPrefixComparator_descriptor, new java.lang.String[] { "Comparable", }); internal_static_BitComparator_descriptor = - getDescriptor().getMessageTypes().get(5); + getDescriptor().getMessageTypes().get(4); internal_static_BitComparator_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_BitComparator_descriptor, new java.lang.String[] { "Comparable", "BitwiseOp", }); internal_static_NullComparator_descriptor = - getDescriptor().getMessageTypes().get(6); + getDescriptor().getMessageTypes().get(5); internal_static_NullComparator_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_NullComparator_descriptor, new java.lang.String[] { }); internal_static_RegexStringComparator_descriptor = - getDescriptor().getMessageTypes().get(7); + getDescriptor().getMessageTypes().get(6); internal_static_RegexStringComparator_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_RegexStringComparator_descriptor, new java.lang.String[] { "Pattern", "PatternFlags", "Charset", "Engine", }); internal_static_SubstringComparator_descriptor = - getDescriptor().getMessageTypes().get(8); + getDescriptor().getMessageTypes().get(7); internal_static_SubstringComparator_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_SubstringComparator_descriptor, diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto index 7e7839564dfe..dbf00dcdaa28 100644 --- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto @@ -110,9 +110,6 @@ message RegionLoad { /** the most recent sequence Id from cache flush */ optional uint64 complete_sequence_id = 15; - - /** The current data locality for region in the regionserver */ - optional float data_locality = 16; } /* Server-level protobufs */ diff --git a/hbase-protocol/src/main/protobuf/Comparator.proto b/hbase-protocol/src/main/protobuf/Comparator.proto index a44ec84632d2..202de8529809 100644 --- a/hbase-protocol/src/main/protobuf/Comparator.proto +++ b/hbase-protocol/src/main/protobuf/Comparator.proto @@ -39,10 +39,6 @@ message BinaryComparator { required ByteArrayComparable comparable = 1; } -message LongComparator { - required ByteArrayComparable comparable = 1; -} - message BinaryPrefixComparator { required ByteArrayComparable comparable = 1; } diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml index 9b3fa63af4ce..648e5f66252c 100644 --- a/hbase-rest/pom.xml +++ b/hbase-rest/pom.xml @@ -233,7 +233,7 @@ - skipRestTests + skipRestTets skipRestTests diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp index 43895a88072b..810569c13cb1 100644 --- a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp @@ -86,7 +86,7 @@ String listenPort = conf.get("hbase.rest.port", "8080"); HBase Version - <%= VersionInfo.getVersion() %>, revision=<%= VersionInfo.getRevision() %> + <%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %> HBase version and revision diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 72aadb1715d9..33883c253195 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -53,27 +53,13 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - + + org.apache.maven.plugins + maven-site-plugin + + true + + diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 2f0451c0f579..d75406ea7cfa 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -225,7 +225,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); HBase Version - <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision + <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision HBase Compiled @@ -239,7 +239,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); Hadoop Version - <% org.apache.hadoop.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.util.VersionInfo.getRevision() %> + <% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %> Hadoop version and revision diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 8ee1193c283c..68b4b7d57f93 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -132,7 +132,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; HBase Version - <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %> + <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %> HBase version and revision @@ -145,16 +145,6 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; <% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %> HBase source MD5 checksum - - Hadoop Version - <% org.apache.hadoop.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.util.VersionInfo.getRevision() %> - Hadoop version and revision - - - Hadoop Compiled - <% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %> - When Hadoop version was compiled and by whom - Zookeeper Quorum <% regionServer.getZooKeeper().getQuorum() %> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index 6ca8ec6dd72b..3ff4cb6dd51f 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -154,7 +154,6 @@ Storefile Size Index Size Bloom Size - Data Locality <%for HRegionInfo r: onlineRegions %> @@ -172,7 +171,6 @@ <% load.getStorefileSizeMB() %>m <% load.getTotalStaticIndexSizeKB() %>k <% load.getTotalStaticBloomSizeKB() %>k - <% load.getDataLocality() %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java index 7cc3f6d5adfe..e7e78328a561 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/CoordinatedStateManagerFactory.java @@ -27,12 +27,7 @@ * based on configuration. */ @InterfaceAudience.Private -public final class CoordinatedStateManagerFactory { - - /** - * Private to keep this class from being accidentally instantiated. - */ - private CoordinatedStateManagerFactory(){} +public class CoordinatedStateManagerFactory { /** * Creates consensus provider from the given configuration. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java index 11da20f66aa1..d621cbf2a930 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/DaemonThreadFactory.java @@ -24,10 +24,10 @@ * Thread factory that creates daemon threads */ public class DaemonThreadFactory implements ThreadFactory { - private static final AtomicInteger poolNumber = new AtomicInteger(1); - private final ThreadGroup group; - private final AtomicInteger threadNumber = new AtomicInteger(1); - private final String namePrefix; + static final AtomicInteger poolNumber = new AtomicInteger(1); + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; public DaemonThreadFactory(String name) { SecurityManager s = System.getSecurityManager(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java index 8d65c66891d4..4226c3f5cac7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HealthCheckChore.java @@ -58,7 +58,7 @@ protected void chore() { if (!isHealthy) { boolean needToStop = decideToStop(); if (needToStop) { - this.getStopper().stop("The node reported unhealthy " + threshold + this.stopper.stop("The node reported unhealthy " + threshold + " number of times consecutively."); } // Always log health report. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java index 6a115ba7f4af..7a1ea11d6b85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java @@ -51,7 +51,7 @@ public class JMXListener implements Coprocessor { public static final Log LOG = LogFactory.getLog(JMXListener.class); public static final String RMI_REGISTRY_PORT_CONF_KEY = ".rmi.registry.port"; public static final String RMI_CONNECTOR_PORT_CONF_KEY = ".rmi.connector.port"; - public static final int defRMIRegistryPort = 10102; + public static int defRMIRegistryPort = 10102; /** * workaround for HBASE-11146 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java index c7bfd03e9595..29a7055aa063 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/TableDescriptors.java @@ -93,14 +93,4 @@ void add(final TableDescriptor htd) */ HTableDescriptor remove(final TableName tablename) throws IOException; - - /** - * Enables the tabledescriptor cache - */ - void setCacheOn() throws IOException; - - /** - * Disables the tabledescriptor cache - */ - void setCacheOff() throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/TagRewriteCell.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/TagRewriteCell.java deleted file mode 100644 index 313ecb8b5862..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/TagRewriteCell.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import java.io.IOException; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.util.ClassSize; - -/** - * This can be used when a Cell has to change with addition/removal of one or more tags. This is an - * efficient way to do so in which only the tags bytes part need to recreated and copied. All other - * parts, refer to the original Cell. - */ -@InterfaceAudience.Private -public class TagRewriteCell implements Cell, SettableSequenceId, SettableTimestamp, HeapSize { - - private Cell cell; - private byte[] tags; - - /** - * @param cell The original Cell which it rewrites - * @param tags the tags bytes. The array suppose to contain the tags bytes alone. - */ - public TagRewriteCell(Cell cell, byte[] tags) { - assert cell instanceof SettableSequenceId; - assert cell instanceof SettableTimestamp; - this.cell = cell; - this.tags = tags; - // tag offset will be treated as 0 and length this.tags.length - if (this.cell instanceof TagRewriteCell) { - // Cleaning the ref so that the byte[] can be GCed - ((TagRewriteCell) this.cell).tags = null; - } - } - - @Override - public byte[] getRowArray() { - return cell.getRowArray(); - } - - @Override - public int getRowOffset() { - return cell.getRowOffset(); - } - - @Override - public short getRowLength() { - return cell.getRowLength(); - } - - @Override - public byte[] getFamilyArray() { - return cell.getFamilyArray(); - } - - @Override - public int getFamilyOffset() { - return cell.getFamilyOffset(); - } - - @Override - public byte getFamilyLength() { - return cell.getFamilyLength(); - } - - @Override - public byte[] getQualifierArray() { - return cell.getQualifierArray(); - } - - @Override - public int getQualifierOffset() { - return cell.getQualifierOffset(); - } - - @Override - public int getQualifierLength() { - return cell.getQualifierLength(); - } - - @Override - public long getTimestamp() { - return cell.getTimestamp(); - } - - @Override - public byte getTypeByte() { - return cell.getTypeByte(); - } - - @Override - @Deprecated - public long getMvccVersion() { - return getSequenceId(); - } - - @Override - public long getSequenceId() { - return cell.getSequenceId(); - } - - @Override - public byte[] getValueArray() { - return cell.getValueArray(); - } - - @Override - public int getValueOffset() { - return cell.getValueOffset(); - } - - @Override - public int getValueLength() { - return cell.getValueLength(); - } - - @Override - public byte[] getTagsArray() { - return this.tags; - } - - @Override - public int getTagsOffset() { - return 0; - } - - @Override - public int getTagsLength() { - return this.tags.length; - } - - @Override - @Deprecated - public byte[] getValue() { - return cell.getValue(); - } - - @Override - @Deprecated - public byte[] getFamily() { - return cell.getFamily(); - } - - @Override - @Deprecated - public byte[] getQualifier() { - return cell.getQualifier(); - } - - @Override - @Deprecated - public byte[] getRow() { - return cell.getRow(); - } - - @Override - public long heapSize() { - long sum = CellUtil.estimatedHeapSizeOf(cell) - cell.getTagsLength(); - sum += ClassSize.OBJECT;// this object itself - sum += (2 * ClassSize.REFERENCE);// pointers to cell and tags array - if (this.tags != null) { - sum += ClassSize.align(ClassSize.ARRAY);// "tags" - sum += this.tags.length; - } - return sum; - } - - @Override - public void setTimestamp(long ts) throws IOException { - // The incoming cell is supposed to be SettableTimestamp type. - CellUtil.setTimestamp(cell, ts); - } - - @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { - // The incoming cell is supposed to be SettableTimestamp type. - CellUtil.setTimestamp(cell, ts, tsOffset); - } - - @Override - public void setSequenceId(long seqId) throws IOException { - // The incoming cell is supposed to be SettableSequenceId type. - CellUtil.setSequenceId(cell, seqId); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java index a1dcd2fe368c..46ee83c23f56 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientSideRegionScanner.java @@ -86,7 +86,7 @@ public Result next() throws IOException { if (this.scanMetrics != null) { long resultSize = 0; for (Cell cell : values) { - resultSize += CellUtil.estimatedSerializedSizeOf(cell); + resultSize += CellUtil.estimatedLengthOf(cell); } this.scanMetrics.countOfBytesInResults.addAndGet(resultSize); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTableWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTableWrapper.java index 660733d62258..afc8a090cfaa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTableWrapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTableWrapper.java @@ -238,7 +238,6 @@ public void batch(List actions, Object[] results) * @deprecated If any exception is thrown by one of the actions, there is no way to * retrieve the partially executed results. Use {@link #batch(List, Object[])} instead. */ - @Deprecated @Override public Object[] batch(List actions) throws IOException, InterruptedException { @@ -258,7 +257,6 @@ public void batchCallback(List actions, Object[] results, * {@link #batchCallback(List, Object[], org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} * instead. */ - @Deprecated @Override public Object[] batchCallback(List actions, Batch.Callback callback) throws IOException, InterruptedException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java index 0f8baa3691ff..0fb5c598d4f2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java @@ -98,6 +98,8 @@ public static class ZkSplitLogManagerDetails extends SplitLogManagerDetails { SplitLogManagerDetails details; + private final Stoppable stopper = null; + // When lastRecoveringNodeCreationTime is older than the following threshold, we'll check // whether to GC stale recovering znodes private volatile long lastRecoveringNodeCreationTime = 0; @@ -667,7 +669,7 @@ public void markRegionsRecovering(final ServerName serverName, Set } catch (InterruptedException e) { throw new InterruptedIOException(); } - } while ((--retries) > 0); + } while ((--retries) > 0 && (!this.stopper.isStopped())); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java deleted file mode 100644 index c7f0b901a69d..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BulkLoadObserver.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.coprocessor; - -import java.io.IOException; - -import org.apache.hadoop.hbase.Coprocessor; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadRequest; -import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadRequest; - -/** - * Coprocessors implement this interface to observe and mediate bulk load operations. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) -@InterfaceStability.Evolving -public interface BulkLoadObserver extends Coprocessor { - /** - * Called as part of SecureBulkLoadEndpoint.prepareBulkLoad() RPC call. - * It can't bypass the default action, e.g., ctx.bypass() won't have effect. - * @param ctx the environment to interact with the framework and master - * @throws IOException - */ - void prePrepareBulkLoad(ObserverContext ctx, - PrepareBulkLoadRequest request) throws IOException; - - /** - * Called as part of SecureBulkLoadEndpoint.cleanupBulkLoad() RPC call. - * It can't bypass the default action, e.g., ctx.bypass() won't have effect. - * @param ctx the environment to interact with the framework and master - * @throws IOException - */ - void preCleanupBulkLoad(ObserverContext ctx, - CleanupBulkLoadRequest request) throws IOException; -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index 17fcabc58b70..c9bd0b60a367 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -280,26 +280,6 @@ public Coprocessor findCoprocessor(String className) { return null; } - /** - * Find list of coprocessors that extend/implement the given class/interface - * @param cls the class/interface to look for - * @return the list of coprocessors, or null if not found - */ - public List findCoprocessors(Class cls) { - ArrayList ret = new ArrayList(); - - for (E env: coprocessors) { - Coprocessor cp = env.getInstance(); - - if(cp != null) { - if (cls.isAssignableFrom(cp.getClass())) { - ret.add((T)cp); - } - } - } - return ret; - } - /** * Find a coprocessor environment by class name * @param className the class name diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java index e526d63278cd..474e3981ef2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java @@ -118,7 +118,6 @@ InternalScanner preFlushScannerOpen(final ObserverContext c) throws IOException; /** @@ -139,7 +138,6 @@ InternalScanner preFlush(final ObserverContext c, * @throws IOException if an error occurred on the coprocessor * @deprecated use {@link #preFlush(ObserverContext, Store, InternalScanner)} instead. */ - @Deprecated void postFlush(final ObserverContext c) throws IOException; /** @@ -341,7 +339,6 @@ void postCompact(final ObserverContext c, final St * @deprecated Use preSplit( * final ObserverContext c, byte[] splitRow) */ - @Deprecated void preSplit(final ObserverContext c) throws IOException; /** @@ -362,7 +359,6 @@ void preSplit(final ObserverContext c, byte[] spli * @throws IOException if an error occurred on the coprocessor * @deprecated Use postCompleteSplit() instead */ - @Deprecated void postSplit(final ObserverContext c, final HRegion l, final HRegion r) throws IOException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java index 49d84ed63272..70c416a35355 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALObserver.java @@ -33,9 +33,6 @@ * It's provided to have a way for coprocessors to observe, rewrite, * or skip WALEdits as they are being written to the WAL. * - * Note that implementers of WALObserver will not see WALEdits that report themselves - * as empty via {@link WALEdit#isEmpty()}. - * * {@link org.apache.hadoop.hbase.coprocessor.RegionObserver} provides * hooks for adding logic for WALEdits in the region context during reconstruction, * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java index b7cab0fa332c..f7340bfd4f25 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java @@ -337,33 +337,22 @@ public Path[] getLocations() { public String toString() { StringBuilder str = new StringBuilder(getClass().getName()); str.append(" locations=["); - for (int i = 0; i < locations.length; ++i) { - if (i > 0) str.append(", "); - str.append(locations[i].toString()); + int i = 0; + for (Path location: locations) { + if (i++ > 0) str.append(", "); + str.append(location.toString()); } str.append("]"); return str.toString(); } - /** - * @return true if the file pointed by the link exists - */ - public boolean exists(final FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { - if (fs.exists(locations[i])) { - return true; - } - } - return false; - } - /** * @return the path of the first available link. */ public Path getAvailablePath(FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { - if (fs.exists(locations[i])) { - return locations[i]; + for (Path path: locations) { + if (fs.exists(path)) { + return path; } } throw new FileNotFoundException("Unable to open link: " + this); @@ -377,9 +366,9 @@ public Path getAvailablePath(FileSystem fs) throws IOException { * @throws IOException on unexpected error. */ public FileStatus getFileStatus(FileSystem fs) throws IOException { - for (int i = 0; i < locations.length; ++i) { + for (Path path: locations) { try { - return fs.getFileStatus(locations[i]); + return fs.getFileStatus(path); } catch (FileNotFoundException e) { // Try another file location } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index 2ef59d17f82f..3926759f45b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -245,6 +245,15 @@ public static TableName getReferencedTableName(final String fileName) { return(TableName.valueOf(m.group(1), m.group(2))); } + /** + * Returns true if the HFileLink exists + */ + public boolean exists(final FileSystem fs) throws IOException { + return fs.exists(this.originPath) || + fs.exists(this.tempPath) || + fs.exists(this.archivePath); + } + /** * Create a new HFileLink name * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java index 2bef680e13bd..1efcd474a97c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java @@ -24,17 +24,17 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; @@ -51,6 +51,8 @@ public abstract class AbstractHFileWriter implements HFile.Writer { /** The Cell previously appended. Becomes the last cell in the file.*/ protected Cell lastCell = null; + protected int lastKeyLength = -1; + /** FileSystem stream to write into. */ protected FSDataOutputStream outputStream; @@ -81,11 +83,8 @@ public abstract class AbstractHFileWriter implements HFile.Writer { /** {@link Writable}s representing meta block data. */ protected List metaData = new ArrayList(); - /** - * First cell in a block. - * This reference should be short-lived since we write hfiles in a burst. - */ - protected Cell firstCellInBlock = null; + /** First key in a block. */ + protected byte[] firstKeyInBlock = null; /** May be null if we were passed a stream. */ protected final Path path; @@ -135,7 +134,8 @@ protected void finishFileInfo() throws IOException { if (lastCell != null) { // Make a copy. The copy is stuffed into our fileinfo map. Needs a clean // byte buffer. Won't take a tuple. - byte [] lastKey = CellUtil.getCellKeySerializedAsKeyValueKey(this.lastCell); + byte[] lastKey = new byte[lastKeyLength]; + KeyValueUtil.appendKeyTo(lastCell, lastKey, 0); fileInfo.append(FileInfo.LASTKEY, lastKey, false); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index f212f14f45c1..acb1ef525483 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -189,18 +189,18 @@ public CacheConfig(Configuration conf, HColumnDescriptor family) { // For the following flags we enable them regardless of per-schema settings // if they are enabled in the global configuration. conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(), + DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(), conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(), + DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(), conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, - DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(), + DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(), conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, - DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(), + DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, - DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), + DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen(), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, - HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1() + HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.shouldCacheDataInL1() ); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java index f34b83dc1443..328221326b02 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java @@ -23,7 +23,6 @@ import java.util.zip.Checksum; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -133,11 +132,7 @@ static boolean validateBlockChecksum(Path path, HFileBlock block, } // Extract the header and compute checksum for the header. ByteBuffer hdr = block.getBufferWithHeader(); - if (hdr.hasArray()) { - checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize); - } else { - checksumObject.update(ByteBufferUtils.toBytes(hdr, 0, hdrSize), 0, hdrSize); - } + checksumObject.update(hdr.array(), hdr.arrayOffset(), hdrSize); int off = hdrSize; int consumed = hdrSize; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index f938020a479e..2b88f817044e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -341,11 +341,7 @@ public static final WriterFactory getWriterFactory(Configuration conf, } } - /** - * An abstraction used by the block index. - * Implementations will check cache for any asked-for block and return cached block if found. - * Otherwise, after reading from fs, will try and put block into cache before returning. - */ + /** An abstraction used by the block index */ public interface CachingBlockReader { /** * Read in a file block. @@ -354,13 +350,15 @@ public interface CachingBlockReader { * @param cacheBlock * @param pread * @param isCompaction is this block being read as part of a compaction - * @param expectedBlockType the block type we are expecting to read with this read operation, - * or null to read whatever block type is available and avoid checking (that might reduce - * caching efficiency of encoded data blocks) - * @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks - * to be in, or null to not perform this check and return the block irrespective of the - * encoding. This check only applies to data blocks and can be set to null when the caller is - * expecting to read a non-data block and has set expectedBlockType accordingly. + * @param expectedBlockType the block type we are expecting to read with this read operation, or + * null to read whatever block type is available and avoid checking (that might reduce + * caching efficiency of encoded data blocks) + * @param expectedDataBlockEncoding the data block encoding the caller is + * expecting data blocks to be in, or null to not perform this + * check and return the block irrespective of the encoding. This + * check only applies to data blocks and can be set to null when + * the caller is expecting to read a non-data block and has set + * expectedBlockType accordingly. * @return Block wrapped in a ByteBuffer. * @throws IOException */ @@ -382,9 +380,11 @@ public interface Reader extends Closeable, CachingBlockReader { KVComparator getComparator(); - HFileScanner getScanner(boolean cacheBlocks, final boolean pread, final boolean isCompaction); + HFileScanner getScanner(boolean cacheBlocks, + final boolean pread, final boolean isCompaction); - ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock) throws IOException; + ByteBuffer getMetaBlock(String metaBlockName, + boolean cacheBlock) throws IOException; Map loadFileInfo() throws IOException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index c1670febdd4e..6341f2d5a0e1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutput; @@ -34,7 +35,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.fs.HFileSystem; -import org.apache.hadoop.hbase.io.ByteBufferInputStream; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; @@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; @@ -140,7 +139,7 @@ public HFileBlock deserialize(ByteBuffer buf, boolean reuse) throws IOException{ } return ourBuffer; } - + @Override public int getDeserialiserIdentifier() { return deserializerIdentifier; @@ -197,7 +196,7 @@ public HFileBlock deserialize(ByteBuffer b) throws IOException { /** * Creates a new {@link HFile} block from the given fields. This constructor * is mostly used when the block data has already been read and uncompressed, - * and is sitting in a byte buffer. + * and is sitting in a byte buffer. * * @param blockType the type of this block, see {@link BlockType} * @param onDiskSizeWithoutHeader see {@link #onDiskSizeWithoutHeader} @@ -246,7 +245,7 @@ public HFileBlock deserialize(ByteBuffer b) throws IOException { * and takes ownership of the buffer. By definition of rewind, ignores the * buffer position, but if you slice the buffer beforehand, it will rewind * to that point. The reason this has a minorNumber and not a majorNumber is - * because majorNumbers indicate the format of a HFile whereas minorNumbers + * because majorNumbers indicate the format of a HFile whereas minorNumbers * indicate the format inside a HFileBlock. */ HFileBlock(ByteBuffer b, boolean usesHBaseChecksum) throws IOException { @@ -332,10 +331,8 @@ private void overwriteHeader() { * @return the buffer with header skipped and checksum omitted. */ public ByteBuffer getBufferWithoutHeader() { - ByteBuffer dup = this.buf.duplicate(); - dup.position(headerSize()); - dup.limit(buf.limit() - totalChecksumBytes()); - return dup.slice(); + return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + headerSize(), + buf.limit() - headerSize() - totalChecksumBytes()).slice(); } /** @@ -348,21 +345,19 @@ public ByteBuffer getBufferWithoutHeader() { * @return the buffer of this block for read-only operations */ public ByteBuffer getBufferReadOnly() { - ByteBuffer dup = this.buf.duplicate(); - dup.limit(buf.limit() - totalChecksumBytes()); - return dup.slice(); + return ByteBuffer.wrap(buf.array(), buf.arrayOffset(), + buf.limit() - totalChecksumBytes()).slice(); } /** * Returns the buffer of this block, including header data. The clients must * not modify the buffer object. This method has to be public because it is * used in {@link BucketCache} to avoid buffer copy. - * + * * @return the buffer with header and checksum included for read-only operations */ public ByteBuffer getBufferReadOnlyWithHeader() { - ByteBuffer dup = this.buf.duplicate(); - return dup.slice(); + return ByteBuffer.wrap(buf.array(), buf.arrayOffset(), buf.limit()).slice(); } /** @@ -455,22 +450,17 @@ public String toString() { .append("(").append(onDiskSizeWithoutHeader) .append("+").append(HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM).append(")"); } - String dataBegin = null; - if (buf.hasArray()) { - dataBegin = Bytes.toStringBinary(buf.array(), buf.arrayOffset() + headerSize(), - Math.min(32, buf.limit() - buf.arrayOffset() - headerSize())); - } else { - ByteBuffer bufWithoutHeader = getBufferWithoutHeader(); - byte[] dataBeginBytes = new byte[Math.min(32, - bufWithoutHeader.limit() - bufWithoutHeader.position())]; - bufWithoutHeader.get(dataBeginBytes); - dataBegin = Bytes.toStringBinary(dataBeginBytes); - } sb.append(" getOnDiskSizeWithHeader()=").append(getOnDiskSizeWithHeader()) .append(" totalChecksumBytes()=").append(totalChecksumBytes()) .append(" isUnpacked()=").append(isUnpacked()) - .append(" buf=[ ").append(buf).append(" ]") - .append(" dataBeginsWith=").append(dataBegin) + .append(" buf=[ ") + .append(buf) + .append(", array().length=").append(buf.array().length) + .append(", arrayOffset()=").append(buf.arrayOffset()) + .append(" ]") + .append(" dataBeginsWith=") + .append(Bytes.toStringBinary(buf.array(), buf.arrayOffset() + headerSize(), + Math.min(32, buf.limit() - buf.arrayOffset() - headerSize()))) .append(" fileContext=").append(fileContext) .append(" ]"); return sb.toString(); @@ -479,20 +469,13 @@ public String toString() { /** * Called after reading a block with provided onDiskSizeWithHeader. */ - private void validateOnDiskSizeWithoutHeader(int expectedOnDiskSizeWithoutHeader) - throws IOException { + private void validateOnDiskSizeWithoutHeader( + int expectedOnDiskSizeWithoutHeader) throws IOException { if (onDiskSizeWithoutHeader != expectedOnDiskSizeWithoutHeader) { - String dataBegin = null; - if (buf.hasArray()) { - dataBegin = Bytes.toStringBinary(buf.array(), buf.arrayOffset(), Math.min(32, buf.limit())); - } else { - ByteBuffer bufDup = getBufferReadOnly(); - byte[] dataBeginBytes = new byte[Math.min(32, bufDup.limit() - bufDup.position())]; - bufDup.get(dataBeginBytes); - dataBegin = Bytes.toStringBinary(dataBeginBytes); - } String blockInfoMsg = - "Block offset: " + offset + ", data starts with: " + dataBegin; + "Block offset: " + offset + ", data starts with: " + + Bytes.toStringBinary(buf.array(), buf.arrayOffset(), + buf.arrayOffset() + Math.min(32, buf.limit())); throw new IOException("On-disk size without header provided is " + expectedOnDiskSizeWithoutHeader + ", but block " + "header contains " + onDiskSizeWithoutHeader + ". " + @@ -517,30 +500,16 @@ HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException HFileBlockDecodingContext ctx = blockType == BlockType.ENCODED_DATA ? reader.getBlockDecodingContext() : reader.getDefaultBlockDecodingContext(); - - ByteBuffer dup = this.buf.duplicate(); - dup.position(this.headerSize()); - dup = dup.slice(); ctx.prepareDecoding(unpacked.getOnDiskSizeWithoutHeader(), unpacked.getUncompressedSizeWithoutHeader(), unpacked.getBufferWithoutHeader(), - dup); + this.getBufferReadOnlyWithHeader().array(), this.headerSize()); // Preserve the next block's header bytes in the new block if we have them. if (unpacked.hasNextBlockHeader()) { - // Both the buffers are limited till checksum bytes and avoid the next block's header. - // Below call to copyFromBufferToBuffer() will try positional read/write from/to buffers when - // any of the buffer is DBB. So we change the limit on a dup buffer. No copying just create - // new BB objects - ByteBuffer inDup = this.buf.duplicate(); - inDup.limit(inDup.limit() + headerSize()); - ByteBuffer outDup = unpacked.buf.duplicate(); - outDup.limit(outDup.limit() + unpacked.headerSize()); - ByteBufferUtils.copyFromBufferToBuffer( - outDup, - inDup, - this.onDiskDataSizeWithHeader, - unpacked.headerSize() + unpacked.uncompressedSizeWithoutHeader - + unpacked.totalChecksumBytes(), unpacked.headerSize()); + System.arraycopy(this.buf.array(), this.buf.arrayOffset() + this.onDiskDataSizeWithHeader, + unpacked.buf.array(), unpacked.buf.arrayOffset() + unpacked.headerSize() + + unpacked.uncompressedSizeWithoutHeader + unpacked.totalChecksumBytes(), + unpacked.headerSize()); } return unpacked; } @@ -554,7 +523,7 @@ private boolean hasNextBlockHeader() { /** * Always allocates a new buffer of the correct size. Copies header bytes - * from the existing buffer. Does not change header fields. + * from the existing buffer. Does not change header fields. * Reserve room to keep checksum bytes too. */ private void allocateBuffer() { @@ -563,14 +532,11 @@ private void allocateBuffer() { int capacityNeeded = headerSize + uncompressedSizeWithoutHeader + cksumBytes + (hasNextBlockHeader() ? headerSize : 0); - // TODO we need consider allocating offheap here? ByteBuffer newBuf = ByteBuffer.allocate(capacityNeeded); - // Copy header bytes into newBuf. - // newBuf is HBB so no issue in calling array() - ByteBuffer dup = buf.duplicate(); - dup.position(0); - dup.get(newBuf.array(), newBuf.arrayOffset(), headerSize); + // Copy header bytes. + System.arraycopy(buf.array(), buf.arrayOffset(), newBuf.array(), + newBuf.arrayOffset(), headerSize); buf = newBuf; // set limit to exclude next block's header @@ -624,9 +590,8 @@ public long getOffset() { * @return a byte stream reading the data + checksum of this block */ public DataInputStream getByteStream() { - ByteBuffer dup = this.buf.duplicate(); - dup.position(this.headerSize()); - return new DataInputStream(new ByteBufferInputStream(dup)); + return new DataInputStream(new ByteArrayInputStream(buf.array(), + buf.arrayOffset() + headerSize(), buf.limit() - headerSize())); } @Override @@ -813,7 +778,7 @@ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) } baosInMemory = new ByteArrayOutputStream(); - + prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; @@ -898,7 +863,7 @@ void ensureBlockReady() throws IOException { */ private void finishBlock() throws IOException { if (blockType == BlockType.DATA) { - BufferGrabbingByteArrayOutputStream baosInMemoryCopy = + BufferGrabbingByteArrayOutputStream baosInMemoryCopy = new BufferGrabbingByteArrayOutputStream(); baosInMemory.writeTo(baosInMemoryCopy); this.dataBlockEncoder.endBlockEncoding(dataBlockEncodingCtx, userDataStream, @@ -1156,7 +1121,7 @@ public void writeBlock(BlockWritable bw, FSDataOutputStream out) /** * Creates a new HFileBlock. Checksums have already been validated, so * the byte buffer passed into the constructor of this newly created - * block does not have checksum data even though the header minor + * block does not have checksum data even though the header minor * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a * 0 value in bytesPerChecksum. */ @@ -1395,12 +1360,12 @@ protected int readAtOffset(FSDataInputStream istream, private static class PrefetchedHeader { long offset = -1; byte[] header = new byte[HConstants.HFILEBLOCK_HEADER_SIZE]; - final ByteBuffer buf = ByteBuffer.wrap(header, 0, HConstants.HFILEBLOCK_HEADER_SIZE); + ByteBuffer buf = ByteBuffer.wrap(header, 0, HConstants.HFILEBLOCK_HEADER_SIZE); } /** Reads version 2 blocks from the filesystem. */ - static class FSReaderImpl extends AbstractFSReader { - /** The file system stream of the underlying {@link HFile} that + static class FSReaderV2 extends AbstractFSReader { + /** The file system stream of the underlying {@link HFile} that * does or doesn't do checksum validations in the filesystem */ protected FSDataInputStreamWrapper streamWrapper; @@ -1417,7 +1382,7 @@ public PrefetchedHeader initialValue() { } }; - public FSReaderImpl(FSDataInputStreamWrapper stream, long fileSize, HFileSystem hfs, Path path, + public FSReaderV2(FSDataInputStreamWrapper stream, long fileSize, HFileSystem hfs, Path path, HFileContext fileContext) throws IOException { super(fileSize, hfs, path, fileContext); this.streamWrapper = stream; @@ -1431,14 +1396,13 @@ public FSReaderImpl(FSDataInputStreamWrapper stream, long fileSize, HFileSystem * A constructor that reads files with the latest minor version. * This is used by unit tests only. */ - FSReaderImpl(FSDataInputStream istream, long fileSize, HFileContext fileContext) - throws IOException { + FSReaderV2(FSDataInputStream istream, long fileSize, HFileContext fileContext) throws IOException { this(new FSDataInputStreamWrapper(istream), fileSize, null, null, fileContext); } /** - * Reads a version 2 block (version 1 blocks not supported and not expected). Tries to do as - * little memory allocation as possible, using the provided on-disk size. + * Reads a version 2 block. Tries to do as little memory allocation as + * possible, using the provided on-disk size. * * @param offset the offset in the stream to read at * @param onDiskSizeWithHeaderL the on-disk size of the block, including @@ -1449,19 +1413,18 @@ public FSReaderImpl(FSDataInputStreamWrapper stream, long fileSize, HFileSystem */ @Override public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, - int uncompressedSize, boolean pread) - throws IOException { + int uncompressedSize, boolean pread) throws IOException { // get a copy of the current state of whether to validate - // hbase checksums or not for this read call. This is not - // thread-safe but the one constaint is that if we decide - // to skip hbase checksum verification then we are + // hbase checksums or not for this read call. This is not + // thread-safe but the one constaint is that if we decide + // to skip hbase checksum verification then we are // guaranteed to use hdfs checksum verification. boolean doVerificationThruHBaseChecksum = streamWrapper.shouldUseHBaseChecksum(); FSDataInputStream is = streamWrapper.getStream(doVerificationThruHBaseChecksum); - HFileBlock blk = readBlockDataInternal(is, offset, - onDiskSizeWithHeaderL, + HFileBlock blk = readBlockDataInternal(is, offset, + onDiskSizeWithHeaderL, uncompressedSize, pread, doVerificationThruHBaseChecksum); if (blk == null) { @@ -1473,7 +1436,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, if (!doVerificationThruHBaseChecksum) { String msg = "HBase checksum verification failed for file " + path + " at offset " + - offset + " filesize " + fileSize + + offset + " filesize " + fileSize + " but this cannot happen because doVerify is " + doVerificationThruHBaseChecksum; HFile.LOG.warn(msg); @@ -1497,7 +1460,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, path + " at offset " + offset + " filesize " + fileSize); } - } + } if (blk == null && !doVerificationThruHBaseChecksum) { String msg = "readBlockData failed, possibly due to " + "checksum verification failed for file " + path + @@ -1506,7 +1469,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, throw new IOException(msg); } - // If there is a checksum mismatch earlier, then retry with + // If there is a checksum mismatch earlier, then retry with // HBase checksums switched off and use HDFS checksum verification. // This triggers HDFS to detect and fix corrupt replicas. The // next checksumOffCount read requests will use HDFS checksums. @@ -1518,7 +1481,7 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, } /** - * Reads a version 2 block. + * Reads a version 2 block. * * @param offset the offset in the stream to read at * @param onDiskSizeWithHeaderL the on-disk size of the block, including @@ -1526,20 +1489,18 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, * @param uncompressedSize the uncompressed size of the the block. Always * expected to be -1. This parameter is only used in version 1. * @param pread whether to use a positional read - * @param verifyChecksum Whether to use HBase checksums. + * @param verifyChecksum Whether to use HBase checksums. * If HBase checksum is switched off, then use HDFS checksum. * @return the HFileBlock or null if there is a HBase checksum mismatch */ - private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, + private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, long onDiskSizeWithHeaderL, int uncompressedSize, boolean pread, - boolean verifyChecksum) - throws IOException { + boolean verifyChecksum) throws IOException { if (offset < 0) { throw new IOException("Invalid offset=" + offset + " trying to read " + "block (onDiskSize=" + onDiskSizeWithHeaderL + ", uncompressedSize=" + uncompressedSize + ")"); } - if (uncompressedSize != -1) { throw new IOException("Version 2 block reader API does not need " + "the uncompressed size parameter"); @@ -1559,13 +1520,12 @@ private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, // read this block's header as part of the previous read's look-ahead. // And we also want to skip reading the header again if it has already // been read. - // TODO: How often does this optimization fire? Has to be same thread so the thread local - // is pertinent and we have to be reading next block as in a big scan. PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get(); - ByteBuffer headerBuf = prefetchedHeader.offset == offset? prefetchedHeader.buf: null; + ByteBuffer headerBuf = prefetchedHeader.offset == offset ? + prefetchedHeader.buf : null; - // Allocate enough space to fit the next block's header too. int nextBlockOnDiskSize = 0; + // Allocate enough space to fit the next block's header too. byte[] onDiskBlock = null; HFileBlock b = null; @@ -1587,16 +1547,15 @@ private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, if (headerBuf != null) { // the header has been read when reading the previous block, copy // to this block's header - // headerBuf is HBB - assert headerBuf.hasArray(); System.arraycopy(headerBuf.array(), headerBuf.arrayOffset(), onDiskBlock, 0, hdrSize); } else { headerBuf = ByteBuffer.wrap(onDiskBlock, 0, hdrSize); } - // We know the total on-disk size but not the uncompressed size. Parse the header. + // We know the total on-disk size but not the uncompressed size. Read + // the entire block into memory, then parse the header. Here we have + // already read the block's header try { - // TODO: FIX!!! Expensive parse just to get a length b = new HFileBlock(headerBuf, fileContext.isUseHBaseChecksum()); } catch (IOException ex) { // Seen in load testing. Provide comprehensive debug info. @@ -1632,14 +1591,11 @@ private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, // in a series of reads or a random read, and we don't have access // to the block index. This is costly and should happen very rarely. headerBuf = ByteBuffer.allocate(hdrSize); - // headerBuf is HBB readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize, false, offset, pread); } - // TODO: FIX!!! Expensive parse just to get a length b = new HFileBlock(headerBuf, fileContext.isUseHBaseChecksum()); onDiskBlock = new byte[b.getOnDiskSizeWithHeader() + hdrSize]; - // headerBuf is HBB System.arraycopy(headerBuf.array(), headerBuf.arrayOffset(), onDiskBlock, 0, hdrSize); nextBlockOnDiskSize = readAtOffset(is, onDiskBlock, hdrSize, b.getOnDiskSizeWithHeader() @@ -1667,7 +1623,8 @@ private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset, // Set prefetched header if (b.hasNextBlockHeader()) { prefetchedHeader.offset = offset + b.getOnDiskSizeWithHeader(); - System.arraycopy(onDiskBlock, onDiskSizeWithHeader, prefetchedHeader.header, 0, hdrSize); + System.arraycopy(onDiskBlock, onDiskSizeWithHeader, + prefetchedHeader.header, 0, hdrSize); } b.offset = offset; @@ -1712,7 +1669,7 @@ public void closeStreams() throws IOException { @Override public String toString() { - return "hfs=" + hfs + ", path=" + path + ", fileContext=" + fileContext; + return "FSReaderV2 [ hfs=" + hfs + " path=" + path + " fileContext=" + fileContext + " ]"; } } @@ -1728,8 +1685,9 @@ public int getSerializedLength() { @Override public void serialize(ByteBuffer destination) { - ByteBufferUtils.copyFromBufferToBuffer(destination, this.buf, 0, getSerializedLength() - - EXTRA_SERIALIZATION_SPACE); + // assumes HeapByteBuffer + destination.put(this.buf.array(), this.buf.arrayOffset(), + getSerializedLength() - EXTRA_SERIALIZATION_SPACE); serializeExtraInfo(destination); } @@ -1777,8 +1735,9 @@ public boolean equals(Object comparison) { if (castedComparison.uncompressedSizeWithoutHeader != this.uncompressedSizeWithoutHeader) { return false; } - if (ByteBufferUtils.compareTo(this.buf, 0, this.buf.limit(), castedComparison.buf, 0, - castedComparison.buf.limit()) != 0) { + if (Bytes.compareTo(this.buf.array(), this.buf.arrayOffset(), this.buf.limit(), + castedComparison.buf.array(), castedComparison.buf.arrayOffset(), + castedComparison.buf.limit()) != 0) { return false; } return true; @@ -1804,7 +1763,7 @@ int getOnDiskDataSizeWithHeader() { return this.onDiskDataSizeWithHeader; } - /** + /** * Calcuate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ @@ -1867,20 +1826,28 @@ public HFileContext getHFileContext() { * has minor version > 0. */ static String toStringHeader(ByteBuffer buf) throws IOException { - byte[] magicBuf = new byte[Math.min(buf.limit() - buf.position(), BlockType.MAGIC_LENGTH)]; - buf.get(magicBuf); - BlockType bt = BlockType.parse(magicBuf, 0, BlockType.MAGIC_LENGTH); - int compressedBlockSizeNoHeader = buf.getInt();; - int uncompressedBlockSizeNoHeader = buf.getInt();; - long prevBlockOffset = buf.getLong(); - byte cksumtype = buf.get(); - long bytesPerChecksum = buf.getInt(); - long onDiskDataSizeWithHeader = buf.getInt(); - return " Header dump: magic: " + Bytes.toString(magicBuf) + + int offset = buf.arrayOffset(); + byte[] b = buf.array(); + long magic = Bytes.toLong(b, offset); + BlockType bt = BlockType.read(buf); + offset += Bytes.SIZEOF_LONG; + int compressedBlockSizeNoHeader = Bytes.toInt(b, offset); + offset += Bytes.SIZEOF_INT; + int uncompressedBlockSizeNoHeader = Bytes.toInt(b, offset); + offset += Bytes.SIZEOF_INT; + long prevBlockOffset = Bytes.toLong(b, offset); + offset += Bytes.SIZEOF_LONG; + byte cksumtype = b[offset]; + offset += Bytes.SIZEOF_BYTE; + long bytesPerChecksum = Bytes.toInt(b, offset); + offset += Bytes.SIZEOF_INT; + long onDiskDataSizeWithHeader = Bytes.toInt(b, offset); + offset += Bytes.SIZEOF_INT; + return " Header dump: magic: " + magic + " blockType " + bt + - " compressedBlockSizeNoHeader " + + " compressedBlockSizeNoHeader " + compressedBlockSizeNoHeader + - " uncompressedBlockSizeNoHeader " + + " uncompressedBlockSizeNoHeader " + uncompressedBlockSizeNoHeader + " prevBlockOffset " + prevBlockOffset + " checksumType " + ChecksumType.codeToType(cksumtype) + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index d9067cfcc25a..5140181e9f66 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -43,7 +44,6 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader; -import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter; @@ -345,9 +345,10 @@ public byte[] midkey() throws IOException { int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1)); int keyLen = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - keyRelOffset; - int keyOffset = Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset - + SECONDARY_INDEX_ENTRY_OVERHEAD; - targetMidKey = ByteBufferUtils.toBytes(b, keyOffset, keyLen); + int keyOffset = b.arrayOffset() + + Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset + + SECONDARY_INDEX_ENTRY_OVERHEAD; + targetMidKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen); } else { // The middle of the root-level index. targetMidKey = blockKeys[rootCount / 2]; @@ -488,7 +489,9 @@ private byte[] getNonRootIndexedKey(ByteBuffer nonRootIndex, int i) { int targetKeyLength = nonRootIndex.getInt(Bytes.SIZEOF_INT * (i + 2)) - targetKeyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD; - return ByteBufferUtils.toBytes(nonRootIndex, targetKeyOffset, targetKeyLength); + int from = nonRootIndex.arrayOffset() + targetKeyOffset; + int to = from + targetKeyLength; + return Arrays.copyOfRange(nonRootIndex.array(), from, to); } /** @@ -543,8 +546,6 @@ static int binarySearchNonRootIndex(Cell key, ByteBuffer nonRootIndex, // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. - // TODO make KeyOnlyKeyValue to be Buffer backed and avoid array() call. This has to be - // done after HBASE-12224 & HBASE-12282 nonRootIndexKV.setKey(nonRootIndex.array(), nonRootIndex.arrayOffset() + midKeyOffset, midLength); int cmp = comparator.compareOnlyKeyPortion(key, nonRootIndexKV); @@ -1129,7 +1130,8 @@ public BlockType getInlineBlockType() { * format version 2), or the uncompressed size of the data block ( * {@link HFile} format version 1). */ - public void addEntry(byte[] firstKey, long blockOffset, int blockDataSize) { + public void addEntry(byte[] firstKey, long blockOffset, int blockDataSize) + { curInlineChunk.add(firstKey, blockOffset, blockDataSize); ++totalNumEntries; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java index 8c018c6b1b65..0021cf42c50a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java @@ -228,7 +228,6 @@ public int run(String[] args) { processFile(fileName); } catch (IOException ex) { LOG.error("Error reading " + fileName, ex); - System.exit(-2); } } @@ -245,7 +244,6 @@ private void processFile(Path file) throws IOException { FileSystem fs = file.getFileSystem(getConf()); if (!fs.exists(file)) { System.err.println("ERROR, file doesnt exist: " + file); - System.exit(-2); } HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(getConf()), getConf()); @@ -349,8 +347,8 @@ private void scanKeysValues(Path file, KeyValueStatsCollector fileStats, if (CellComparator.compareRows(pCell, cell) > 0) { System.err.println("WARNING, previous row is greater then" + " current row\n\tfilename -> " + file + "\n\tprevious -> " - + CellUtil.getCellKeyAsString(pCell) + "\n\tcurrent -> " - + CellUtil.getCellKeyAsString(cell)); + + CellUtil.getCellKey(pCell) + "\n\tcurrent -> " + + CellUtil.getCellKey(cell)); } } // check if families are consistent @@ -360,13 +358,13 @@ private void scanKeysValues(Path file, KeyValueStatsCollector fileStats, if (!file.toString().contains(fam)) { System.err.println("WARNING, filename does not match kv family," + "\n\tfilename -> " + file + "\n\tkeyvalue -> " - + CellUtil.getCellKeyAsString(cell)); + + CellUtil.getCellKey(cell)); } if (pCell != null && CellComparator.compareFamilies(pCell, cell) != 0) { System.err.println("WARNING, previous kv has different family" + " compared to current key\n\tfilename -> " + file - + "\n\tprevious -> " + CellUtil.getCellKeyAsString(pCell) - + "\n\tcurrent -> " + CellUtil.getCellKeyAsString(cell)); + + "\n\tprevious -> " + CellUtil.getCellKey(pCell) + + "\n\tcurrent -> " + CellUtil.getCellKey(cell)); } } pCell = cell; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index d58ca109a276..f2249aed40ed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -126,8 +126,8 @@ public HFileReaderV2(final Path path, final FixedFileTrailer trailer, trailer.expectMajorVersion(getMajorVersion()); validateMinorVersion(path, trailer.getMinorVersion()); this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer); - HFileBlock.FSReaderImpl fsBlockReaderV2 = - new HFileBlock.FSReaderImpl(fsdis, fileSize, hfs, path, hfileContext); + HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path, + hfileContext); this.fsBlockReader = fsBlockReaderV2; // upcast // Comparator class name is stored in the trailer in version 2. @@ -365,6 +365,28 @@ public ByteBuffer getMetaBlock(String metaBlockName, boolean cacheBlock) } } + /** + * Read in a file block of the given {@link BlockType} and + * {@link DataBlockEncoding}. Unpacks the block as necessary. + * @param dataBlockOffset offset to read. + * @param onDiskBlockSize size of the block + * @param cacheBlock + * @param pread Use positional read instead of seek+read (positional is + * better doing random reads whereas seek+read is better scanning). + * @param isCompaction is this block being read as part of a compaction + * @param expectedBlockType the block type we are expecting to read with this + * read operation, or null to read whatever block type is available + * and avoid checking (that might reduce caching efficiency of + * encoded data blocks) + * @param expectedDataBlockEncoding the data block encoding the caller is + * expecting data blocks to be in, or null to not perform this + * check and return the block irrespective of the encoding. This + * check only applies to data blocks and can be set to null when + * the caller is expecting to read a non-data block and has set + * expectedBlockType accordingly. + * @return Block wrapped in a ByteBuffer. + * @throws IOException + */ @Override public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, final boolean cacheBlock, boolean pread, final boolean isCompaction, @@ -374,16 +396,20 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, if (dataBlockIndexReader == null) { throw new IOException("Block index not loaded"); } - if (dataBlockOffset < 0 || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) { - throw new IOException("Requested block is out of range: " + dataBlockOffset + - ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset()); + if (dataBlockOffset < 0 + || dataBlockOffset >= trailer.getLoadOnOpenDataOffset()) { + throw new IOException("Requested block is out of range: " + + dataBlockOffset + ", lastDataBlockOffset: " + + trailer.getLastDataBlockOffset()); } - - // For any given block from any given file, synchronize reads for said block. + // For any given block from any given file, synchronize reads for said + // block. // Without a cache, this synchronizing is needless overhead, but really // the other choice is to duplicate work (which the cache would prevent you // from doing). + BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset); + boolean useLock = false; IdLock.Entry lockEntry = null; TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock"); @@ -413,7 +439,6 @@ public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize, + dataBlockEncoder.getDataBlockEncoding() + ")"); } } - // Cache-hit. Return! return cachedBlock; } // Carry on, please load. @@ -588,7 +613,7 @@ public int reseekTo(Cell key) throws IOException { (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader .getComparator() .compareOnlyKeyPortion(key, - new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0, + new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0, nextIndexedKey.length)) < 0)) { // The reader shall continue to scan the current data block instead // of querying the @@ -704,7 +729,7 @@ protected HFileBlock readNextDataBlock() throws IOException { return curBlock; } - + public DataBlockEncoding getEffectiveDataBlockEncoding() { return ((HFileReaderV2)reader).getEffectiveEncodingInCache(isCompaction); } @@ -998,7 +1023,7 @@ protected int blockSeek(Cell key, boolean seekBefore) { if (lastKeyValueSize < 0) { throw new IllegalStateException("blockSeek with seekBefore " + "at the first key of the block: key=" - + CellUtil.getCellKeyAsString(key) + + CellUtil.getCellKey(key) + ", blockOffset=" + block.getOffset() + ", onDiskSize=" + block.getOnDiskSizeWithHeader()); } @@ -1286,7 +1311,7 @@ public boolean isFileInfoLoaded() { private void validateMinorVersion(Path path, int minorVersion) { if (minorVersion < MIN_MINOR_VERSION || minorVersion > MAX_MINOR_VERSION) { - String msg = "Minor version for path " + path + + String msg = "Minor version for path " + path + " is expected to be between " + MIN_MINOR_VERSION + " and " + MAX_MINOR_VERSION + " but is found to be " + minorVersion; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java index b28d8c1a875c..e3c92cf495d5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java @@ -297,7 +297,7 @@ protected int blockSeek(Cell key, boolean seekBefore) { if (lastKeyValueSize < 0) { throw new IllegalStateException("blockSeek with seekBefore " + "at the first key of the block: key=" - + CellUtil.getCellKeyAsString(key) + + CellUtil.getCellKey(key) + ", blockOffset=" + block.getOffset() + ", onDiskSize=" + block.getOnDiskSizeWithHeader()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java index 1df8bc22c2a2..f7843402600e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java @@ -27,15 +27,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable; import org.apache.hadoop.hbase.util.BloomFilterWriter; @@ -76,11 +75,8 @@ public class HFileWriterV2 extends AbstractHFileWriter { /** The offset of the last data block or 0 if the file is empty. */ protected long lastDataBlockOffset; - /** - * The last(stop) Cell of the previous data block. - * This reference should be short-lived since we write hfiles in a burst. - */ - private Cell lastCellOfPreviousBlock = null; + /** The last(stop) Key of the previous data block. */ + private byte[] lastKeyOfPreviousBlock = null; /** Additional data items to be written to the "load-on-open" section. */ private List additionalLoadOnOpenData = @@ -162,11 +158,8 @@ private void finishBlock() throws IOException { fsBlockWriter.writeHeaderAndData(outputStream); int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader(); - // Rather than CellComparator, we should be making use of an Interface here with the - // implementation class serialized out to the HFile metadata. TODO. - Cell indexEntry = CellComparator.getMidpoint(lastCellOfPreviousBlock, firstCellInBlock); - dataBlockIndexWriter.addEntry(CellUtil.getCellKeySerializedAsKeyValueKey(indexEntry), - lastDataBlockOffset, onDiskSize); + byte[] indexKey = comparator.calcIndexKey(lastKeyOfPreviousBlock, firstKeyInBlock); + dataBlockIndexWriter.addEntry(indexKey, lastDataBlockOffset, onDiskSize); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (cacheConf.shouldCacheDataOnWrite()) { doCacheOnWrite(lastDataBlockOffset); @@ -212,9 +205,10 @@ private void doCacheOnWrite(long offset) { protected void newBlock() throws IOException { // This is where the next block begins. fsBlockWriter.startWriting(BlockType.DATA); - firstCellInBlock = null; - if (lastCell != null) { - lastCellOfPreviousBlock = lastCell; + firstKeyInBlock = null; + if (lastKeyLength > 0) { + lastKeyOfPreviousBlock = new byte[lastKeyLength]; + KeyValueUtil.appendKeyTo(lastCell, lastKeyOfPreviousBlock, 0); } } @@ -254,6 +248,7 @@ public void appendMetaBlock(String metaBlockName, Writable content) { */ @Override public void append(final Cell cell) throws IOException { + int klength = KeyValueUtil.keyLength(cell); byte[] value = cell.getValueArray(); int voffset = cell.getValueOffset(); int vlength = cell.getValueLength(); @@ -269,19 +264,18 @@ public void append(final Cell cell) throws IOException { fsBlockWriter.write(cell); - totalKeyLength += CellUtil.estimatedSerializedSizeOfKey(cell); + totalKeyLength += klength; totalValueLength += vlength; // Are we the first key in this block? - if (firstCellInBlock == null) { - // If cell is big, block will be closed and this firstCellInBlock reference will only last - // a short while. - firstCellInBlock = cell; + if (firstKeyInBlock == null) { + // Copy the key for use as first key in block. It is put into file index. + firstKeyInBlock = new byte[klength]; + KeyValueUtil.appendKeyTo(cell, firstKeyInBlock, 0); } - // TODO: What if cell is 10MB and we write infrequently? We'll hold on to the cell here - // indefinetly? lastCell = cell; + lastKeyLength = klength; entryCount++; this.maxMemstoreTS = Math.max(this.maxMemstoreTS, cell.getSequenceId()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 6159a2fdaf12..73a0ca2081c9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -268,7 +268,7 @@ public BucketCache(String ioEngineName, long capacity, int blockSize, int[] buck ", capacity=" + StringUtils.byteDesc(capacity) + ", blockSize=" + StringUtils.byteDesc(blockSize) + ", writerThreadNum=" + writerThreadNum + ", writerQLen=" + writerQLen + ", persistencePath=" + - persistencePath + ", bucketAllocator=" + this.bucketAllocator.getClass().getName()); + persistencePath + ", bucketAllocator=" + this.bucketAllocator); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java index 63c4b325c4cf..09a9ea0eac3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerWrapperImpl.java @@ -27,13 +27,9 @@ public class MetricsHBaseServerWrapperImpl implements MetricsHBaseServerWrapper this.server = server; } - private boolean isServerStarted() { - return this.server != null && this.server.isStarted(); - } - @Override public long getTotalQueueSize() { - if (!isServerStarted()) { + if (this.server == null) { return 0; } return server.callQueueSize.get(); @@ -41,7 +37,7 @@ public long getTotalQueueSize() { @Override public int getGeneralQueueLength() { - if (!isServerStarted() || this.server.getScheduler() == null) { + if (this.server == null || this.server.getScheduler() == null) { return 0; } return server.getScheduler().getGeneralQueueLength(); @@ -49,7 +45,7 @@ public int getGeneralQueueLength() { @Override public int getReplicationQueueLength() { - if (!isServerStarted() || this.server.getScheduler() == null) { + if (this.server == null || this.server.getScheduler() == null) { return 0; } return server.getScheduler().getReplicationQueueLength(); @@ -57,7 +53,7 @@ public int getReplicationQueueLength() { @Override public int getPriorityQueueLength() { - if (!isServerStarted() || this.server.getScheduler() == null) { + if (this.server == null || this.server.getScheduler() == null) { return 0; } return server.getScheduler().getPriorityQueueLength(); @@ -65,7 +61,7 @@ public int getPriorityQueueLength() { @Override public int getNumOpenConnections() { - if (!isServerStarted() || this.server.connectionList == null) { + if (this.server == null || this.server.connectionList == null) { return 0; } return server.connectionList.size(); @@ -73,7 +69,7 @@ public int getNumOpenConnections() { @Override public int getActiveRpcHandlerCount() { - if (!isServerStarted() || this.server.getScheduler() == null) { + if (this.server == null || this.server.getScheduler() == null) { return 0; } return server.getScheduler().getActiveRpcHandlerCount(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index 4c7271464dce..e3654cbe4d2f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -146,7 +146,7 @@ * and keep taking while the server is up. * * CallRunner#run executes the call. When done, asks the included Call to put itself on new - * queue for Responder to pull from and return result to client. + * queue for {@link Responder} to pull from and return result to client. * * @see RpcClient */ @@ -685,7 +685,6 @@ public void run() { doAccept(key); } } catch (IOException ignored) { - if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored); } key = null; } @@ -723,9 +722,7 @@ public void run() { try { acceptChannel.close(); selector.close(); - } catch (IOException ignored) { - if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored); - } + } catch (IOException ignored) { } selector= null; acceptChannel= null; @@ -873,28 +870,22 @@ public void run() { /** * Take the list of the connections that want to write, and register them - * in the selector. + * in the selector. */ - private void registerWrites() { + private void registerWrites(){ Iterator it = writingCons.iterator(); - while (it.hasNext()) { + while (it.hasNext()){ Connection c = it.next(); it.remove(); SelectionKey sk = c.channel.keyFor(writeSelector); - try { - if (sk == null) { - try { - c.channel.register(writeSelector, SelectionKey.OP_WRITE, c); - } catch (ClosedChannelException e) { - // ignore: the client went away. - if (LOG.isTraceEnabled()) LOG.trace("ignored", e); - } - } else { - sk.interestOps(SelectionKey.OP_WRITE); + if (sk == null){ + try { + c.channel.register(writeSelector, SelectionKey.OP_WRITE, c); + } catch (ClosedChannelException e) { + // ignore: the client went away. } - } catch (CancelledKeyException e) { - // ignore: the client went away. - if (LOG.isTraceEnabled()) LOG.trace("ignored", e); + } else { + sk.interestOps(SelectionKey.OP_WRITE); } } } @@ -1057,17 +1048,15 @@ private boolean processResponse(final Call call) throws IOException { /** * Process all the responses for this connection * - * @return true if all the calls were processed or that someone else is doing it. - * false if there * is still some work to do. In this case, we expect the caller to - * delay us. + * @return true if all the calls were processed or that someone else is doing it. false if there + * is still some work to do. In this case, we expect the caller to delay us. * @throws IOException */ private boolean processAllResponses(final Connection connection) throws IOException { // We want only one writer on the channel for a connection at a time. connection.responseWriteLock.lock(); try { - for (int i = 0; i < 20; i++) { - // protection if some handlers manage to need all the responder + for (int i = 0; i < 20; i++) { // protection if some handlers manage to need all the responder Call call = connection.responseQueue.pollFirst(); if (call == null) { return true; @@ -1290,7 +1279,8 @@ SaslUtil.SASL_PROPS, new SaslDigestCallbackHandler( secretManager, this)); break; default: - UserGroupInformation current = UserGroupInformation.getCurrentUser(); + UserGroupInformation current = UserGroupInformation + .getCurrentUser(); String fullName = current.getUserName(); if (LOG.isDebugEnabled()) { LOG.debug("Kerberos principal name is " + fullName); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java index 08018ab6dd57..8bb41115a786 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java @@ -127,10 +127,9 @@ public void map(ImmutableBytesWritable row, Result values, if (!thisRowFamilyName.equals(currentFamilyName)) { currentFamilyName = thisRowFamilyName; context.getCounter("CF", thisRowFamilyName).increment(1); - if (1 == context.getCounter("CF", thisRowFamilyName).getValue()) { - context.write(new Text("Total Families Across all Rows"), new IntWritable(1)); - context.write(new Text(thisRowFamilyName), new IntWritable(1)); - } + context.write(new Text("Total Families Across all Rows"), + new IntWritable(1)); + context.write(new Text(thisRowFamilyName), new IntWritable(1)); } String thisRowQualifierName = thisRowFamilyName + separator + Bytes.toStringBinary(CellUtil.cloneQualifier(value)); @@ -242,7 +241,7 @@ private static Filter getRowFilter(String[] args) { @Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); - if (otherArgs.length < 2) { + if (otherArgs.length < 1) { System.err.println("ERROR: Wrong number of parameters: " + args.length); System.err.println("Usage: CellCounter " + "[^[regex pattern] or [Prefix] for row filter]] "); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java index e88d6dff7bf5..f7ff0e9c8020 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java @@ -65,8 +65,7 @@ public class CopyTable extends Configured implements Tool { String peerAddress = null; String families = null; boolean allCells = false; - static boolean shuffle = false; - + boolean bulkload = false; Path bulkloadDir = null; @@ -87,7 +86,7 @@ public Job createSubmittableJob(String[] args) if (!doCommandLine(args)) { return null; } - + Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); job.setJarByClass(CopyTable.class); Scan scan = new Scan(); @@ -99,27 +98,24 @@ public Job createSubmittableJob(String[] args) if (allCells) { scan.setRaw(true); } - if (shuffle) { - job.getConfiguration().set(TableInputFormat.SHUFFLE_MAPS, "true"); - } if (versions >= 0) { scan.setMaxVersions(versions); } - + if (startRow != null) { scan.setStartRow(Bytes.toBytes(startRow)); } - + if (stopRow != null) { scan.setStopRow(Bytes.toBytes(stopRow)); } - + if(families != null) { String[] fams = families.split(","); Map cfRenameMap = new HashMap(); for(String fam : fams) { String sourceCf; - if(fam.contains(":")) { + if(fam.contains(":")) { // fam looks like "sourceCfName:destCfName" String[] srcAndDest = fam.split(":", 2); sourceCf = srcAndDest[0]; @@ -127,21 +123,21 @@ public Job createSubmittableJob(String[] args) cfRenameMap.put(sourceCf, destCf); } else { // fam is just "sourceCf" - sourceCf = fam; + sourceCf = fam; } scan.addFamily(Bytes.toBytes(sourceCf)); } Import.configureCfRenaming(job.getConfiguration(), cfRenameMap); } job.setNumReduceTasks(0); - + if (bulkload) { TableMapReduceUtil.initTableMapperJob(tableName, scan, Import.KeyValueImporter.class, null, null, job); - + // We need to split the inputs by destination tables so that output of Map can be bulk-loaded. TableInputFormat.configureSplitTable(job, TableName.valueOf(dstTableName)); - + FileSystem fs = FileSystem.get(getConf()); Random rand = new Random(); Path root = new Path(fs.getWorkingDirectory(), "copytable"); @@ -152,7 +148,7 @@ public Job createSubmittableJob(String[] args) break; } } - + System.out.println("HFiles will be stored at " + this.bulkloadDir); HFileOutputFormat2.setOutputPath(job, bulkloadDir); try (Connection conn = ConnectionFactory.createConnection(getConf()); @@ -162,11 +158,11 @@ public Job createSubmittableJob(String[] args) } else { TableMapReduceUtil.initTableMapperJob(tableName, scan, Import.Importer.class, null, null, job); - + TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null, null); } - + return job; } @@ -231,19 +227,19 @@ private boolean doCommandLine(final String[] args) { printUsage(null); return false; } - + final String startRowArgKey = "--startrow="; if (cmd.startsWith(startRowArgKey)) { startRow = cmd.substring(startRowArgKey.length()); continue; } - + final String stopRowArgKey = "--stoprow="; if (cmd.startsWith(stopRowArgKey)) { stopRow = cmd.substring(stopRowArgKey.length()); continue; } - + final String startTimeArgKey = "--starttime="; if (cmd.startsWith(startTimeArgKey)) { startTime = Long.parseLong(cmd.substring(startTimeArgKey.length())); @@ -284,17 +280,12 @@ private boolean doCommandLine(final String[] args) { allCells = true; continue; } - + if (cmd.startsWith("--bulkload")) { bulkload = true; continue; } - if (cmd.startsWith("--shuffle")) { - shuffle = true; - continue; - } - if (i == args.length-1) { tableName = cmd; } else { @@ -311,12 +302,12 @@ private boolean doCommandLine(final String[] args) { printUsage("Invalid time range filter: starttime=" + startTime + " > endtime=" + endTime); return false; } - + if (bulkload && peerAddress != null) { printUsage("Remote bulkload is not supported!"); return false; } - + // set dstTableName if necessary if (dstTableName == null) { dstTableName = tableName; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index 8376e8556c8b..855417d66341 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -203,6 +203,8 @@ private void discoverLoadQueue(Deque ret, Path hfofDir) continue; } Path familyDir = stat.getPath(); + // Skip _logs, etc + if (familyDir.getName().startsWith("_")) continue; byte[] family = familyDir.getName().getBytes(); Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); for (Path hfile : hfiles) { @@ -848,6 +850,8 @@ private void createTable(TableName tableName, String dirPath) throws Exception { continue; } Path familyDir = stat.getPath(); + // Skip _logs, etc + if (familyDir.getName().startsWith("_")) continue; byte[] family = familyDir.getName().getBytes(); hcd = new HColumnDescriptor(family); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java index dccaa25fa685..56ae349843ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.mapreduce; import java.io.IOException; -import java.util.Collections; -import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,8 +33,6 @@ import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.StringUtils; @@ -84,8 +80,6 @@ public class TableInputFormat extends TableInputFormatBase public static final String SCAN_CACHEDROWS = "hbase.mapreduce.scan.cachedrows"; /** Set the maximum number of values to return for each call to next(). */ public static final String SCAN_BATCHSIZE = "hbase.mapreduce.scan.batchsize"; - /** Specify if we have to shuffle the map tasks. */ - public static final String SHUFFLE_MAPS = "hbase.mapreduce.inputtable.shufflemaps"; /** The configuration. */ private Configuration conf = null; @@ -119,7 +113,7 @@ public void setConf(Configuration configuration) { } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } - + Scan scan = null; if (conf.get(SCAN) != null) { @@ -179,7 +173,7 @@ public void setConf(Configuration configuration) { setScan(scan); } - + /** * Parses a combined family and qualifier and adds either both or just the * family in case there is no qualifier. This assumes the older colon @@ -216,25 +210,6 @@ public static void addColumns(Scan scan, byte [][] columns) { } } - /** - * Calculates the splits that will serve as input for the map tasks. The - * number of splits matches the number of regions in a table. Splits are shuffled if - * required. - * @param context The current job context. - * @return The list of input splits. - * @throws IOException When creating the list of splits fails. - * @see org.apache.hadoop.mapreduce.InputFormat#getSplits( - * org.apache.hadoop.mapreduce.JobContext) - */ - @Override - public List getSplits(JobContext context) throws IOException { - List splits = super.getSplits(context); - if ((conf.get(SHUFFLE_MAPS) != null) && "true".equals(conf.get(SHUFFLE_MAPS).toLowerCase())) { - Collections.shuffle(splits); - } - return splits; - } - /** * Convenience method to parse a string representation of an array of column specifiers. * @@ -260,7 +235,7 @@ protected Pair getStartEndKeys() throws IOException { return super.getStartEndKeys(); } - + /** * Sets split table in map-reduce job. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index f69be5012300..268f2b18306a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -20,6 +20,8 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.URL; import java.net.URLDecoder; import java.util.ArrayList; @@ -47,6 +49,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.mapreduce.hadoopbackport.JarFinder; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.security.User; @@ -833,7 +836,8 @@ public static void addDependencyJars(Configuration conf, } /** - * Finds the Jar for a class or creates it if it doesn't exist. If the class is in + * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop), finds + * the Jar for a class or creates it if it doesn't exist. If the class is in * a directory in the classpath, it creates a Jar on the fly with the * contents of the directory and returns the path to that Jar. If a Jar is * created, it is created in the system temporary directory. Otherwise, @@ -927,16 +931,29 @@ private static String findContainingJar(Class my_class, Map p } /** - * Invoke 'getJar' on a custom JarFinder implementation. Useful for some job - * configuration contexts (HBASE-8140) and also for testing on MRv2. - * check if we have HADOOP-9426. + * Invoke 'getJar' on a JarFinder implementation. Useful for some job + * configuration contexts (HBASE-8140) and also for testing on MRv2. First + * check if we have HADOOP-9426. Lacking that, fall back to the backport. * @param my_class the class to find. * @return a jar file that contains the class, or null. */ private static String getJar(Class my_class) { String ret = null; + String hadoopJarFinder = "org.apache.hadoop.util.JarFinder"; + Class jarFinder = null; try { + LOG.debug("Looking for " + hadoopJarFinder + "."); + jarFinder = Class.forName(hadoopJarFinder); + LOG.debug(hadoopJarFinder + " found."); + Method getJar = jarFinder.getMethod("getJar", Class.class); + ret = (String) getJar.invoke(null, my_class); + } catch (ClassNotFoundException e) { + LOG.debug("Using backported JarFinder."); ret = JarFinder.getJar(my_class); + } catch (InvocationTargetException e) { + // function was properly called, but threw it's own exception. Unwrap it + // and pass it on. + throw new RuntimeException(e.getCause()); } catch (Exception e) { // toss all other exceptions, related to reflection failure throw new RuntimeException("getJar invocation failed.", e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java index da40b2ecdf50..52b8e450a245 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java @@ -29,8 +29,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Mutation; @@ -48,7 +46,7 @@ * while the output value must be either a {@link Put} or a * {@link Delete} instance. * - *

is the type of the key. Ignored in this class. + * @param The type of the key. Ignored in this class. */ @InterfaceAudience.Public @InterfaceStability.Stable @@ -81,17 +79,28 @@ public class TableOutputFormat extends OutputFormat /** The configuration. */ private Configuration conf = null; - private Table table; - private Connection connection; + private HTable table; /** * Writes the reducer output to an HBase table. * * @param The type of the key. */ - protected class TableRecordWriter + protected static class TableRecordWriter extends RecordWriter { + /** The table to write to. */ + private Table table; + + /** + * Instantiate a TableRecordWriter with the HBase HClient for writing. + * + * @param table The table to write to. + */ + public TableRecordWriter(Table table) { + this.table = table; + } + /** * Closes the writer, in this case flush table commits. * @@ -103,7 +112,6 @@ protected class TableRecordWriter public void close(TaskAttemptContext context) throws IOException { table.close(); - connection.close(); } /** @@ -117,8 +125,8 @@ public void close(TaskAttemptContext context) @Override public void write(KEY key, Mutation value) throws IOException { - if (value instanceof Put) table.put(new Put((Put)value)); - else if (value instanceof Delete) table.delete(new Delete((Delete)value)); + if (value instanceof Put) this.table.put(new Put((Put)value)); + else if (value instanceof Delete) this.table.delete(new Delete((Delete)value)); else throw new IOException("Pass a Delete or a Put"); } } @@ -136,7 +144,7 @@ public void write(KEY key, Mutation value) public RecordWriter getRecordWriter( TaskAttemptContext context) throws IOException, InterruptedException { - return new TableRecordWriter(); + return new TableRecordWriter(this.table); } /** @@ -197,9 +205,8 @@ public void setConf(Configuration otherConf) { if (zkClientPort != 0) { this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort); } - this.connection = ConnectionFactory.createConnection(this.conf); - this.table = connection.getTable(TableName.valueOf(tableName)); - ((HTable) this.table).setAutoFlush(false, true); + this.table = new HTable(this.conf, TableName.valueOf(tableName)); + this.table.setAutoFlush(false, true); LOG.info("Created table instance for " + tableName); } catch(IOException e) { LOG.error(e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/JarFinder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java similarity index 98% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/JarFinder.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java index 5f1156ec89de..b81ccd4094d5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/JarFinder.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/JarFinder.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. See accompanying LICENSE file. */ -package org.apache.hadoop.hbase.mapreduce; +package org.apache.hadoop.hbase.mapreduce.hadoopbackport; import com.google.common.base.Preconditions; @@ -36,7 +36,7 @@ * classpath, it creates a Jar on the fly with the contents of the directory * and returns the path to that Jar. If a Jar is created, it is created in * the system temporary directory. - * + * * This file was forked from hadoop/common/branches/branch-2@1377176. */ public class JarFinder { @@ -161,7 +161,6 @@ else if ("file".equals(url.getProtocol())) { } File tempJar = File.createTempFile("hadoop-", "", testDir); tempJar = new File(tempJar.getAbsolutePath() + ".jar"); - tempJar.deleteOnExit(); createJar(baseDir, tempJar); return tempJar.getAbsolutePath(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 8c3027ae4853..507d31d67927 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -215,8 +215,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { MasterCoprocessorHost cpHost; - private final boolean preLoadTableDescriptors; - // Time stamps for when a hmaster became active private long masterActiveTime; @@ -264,7 +262,7 @@ public void doGet(HttpServletRequest request, * *

* Remaining steps of initialization occur in - * #finishActiveMasterInitialization(MonitoredTask) after + * {@link #finishActiveMasterInitialization(MonitoredTask)} after * the master becomes the active one. * * @throws KeeperException @@ -292,11 +290,7 @@ public HMaster(final Configuration conf, CoordinatedStateManager csm) this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this)); - // preload table descriptor at startup - this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true); - // Do we publish the status? - boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED, HConstants.STATUS_PUBLISHED_DEFAULT); Class publisherClass = @@ -522,15 +516,6 @@ private void finishActiveMasterInitialization(MonitoredTask status) // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring. this.fileSystemManager = new MasterFileSystem(this, this); - // enable table descriptors cache - this.tableDescriptors.setCacheOn(); - - // warm-up HTDs cache on master initialization - if (preLoadTableDescriptors) { - status.setStatus("Pre-loading table descriptors"); - this.tableDescriptors.getAll(); - } - // publish cluster ID status.setStatus("Publishing Cluster ID in ZooKeeper"); ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId()); @@ -1282,10 +1267,8 @@ private void startActiveMasterManager() throws KeeperException { * this node explicitly. If we crash before then, ZooKeeper will delete * this node for us since it is ephemeral. */ - LOG.info("Adding backup master ZNode " + backupZNode); - if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName)) { - LOG.warn("Failed create of " + backupZNode + " by " + serverName); - } + LOG.info("Adding ZNode for " + backupZNode + " in backup master directory"); + MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName); activeMasterManager = new ActiveMasterManager(zooKeeper, serverName, this); // Start a thread to try to become the active master, so we won't block here diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java index f1606f11ef65..d01e61819f24 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java @@ -166,10 +166,7 @@ private int startMaster() { // login the zookeeper server principal (if using security) ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file", "hbase.zookeeper.server.kerberos.principal", null); - int localZKClusterSessionTimeout = - conf.getInt(HConstants.ZK_SESSION_TIMEOUT + ".localHBaseCluster", 10*1000); - conf.setInt(HConstants.ZK_SESSION_TIMEOUT, localZKClusterSessionTimeout); - LOG.info("Starting a zookeeper cluster"); + int clientPort = zooKeeperCluster.startup(zkDataPath); if (clientPort != zkClientPort) { String errorMsg = "Could not start ZK at requested port of " + @@ -179,15 +176,13 @@ private int startMaster() { System.err.println(errorMsg); throw new IOException(errorMsg); } - conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, + Integer.toString(clientPort)); + conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 10 *1000); // Need to have the zk cluster shutdown when master is shutdown. // Run a subclass that does the zk cluster shutdown on its way out. - int mastersCount = conf.getInt("hbase.masters", 1); - int regionServersCount = conf.getInt("hbase.regionservers", 1); - LOG.info("Starting up instance of localHBaseCluster; master=" + mastersCount + - ", regionserversCount=" + regionServersCount); - LocalHBaseCluster cluster = new LocalHBaseCluster(conf, mastersCount, regionServersCount, - LocalHMaster.class, HRegionServer.class); + LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1), + conf.getInt("hbase.regionservers", 1), LocalHMaster.class, HRegionServer.class); ((LocalHMaster)cluster.getMaster(0)).setZKCluster(zooKeeperCluster); cluster.startup(); waitOnMasterThreads(cluster); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 115cc3539dae..f57fceec4647 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -621,7 +621,7 @@ public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd) throw new InvalidFamilyOperationException("Family '" + Bytes.toString(familyName) + "' doesn't exists so cannot be modified"); } - htd.modifyFamily(hcd); + htd.addFamily(hcd); this.services.getTableDescriptors().add(htd); return htd; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 0a2d7b081cd0..a5e110b6d001 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -526,7 +526,7 @@ public String toString() { @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NM_FIELD_NAMING_CONVENTION", justification="Mistake. Too disruptive to change now") - public static final Action NullAction = new Action(Type.NULL); + public static Action NullAction = new Action(Type.NULL); public void doAction(Action action) { switch (action.type) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java index 0d94bc963249..6cd5b05adced 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java @@ -226,7 +226,7 @@ private boolean checkAndDeleteFiles(List files) { Iterable deletableValidFiles = validFiles; // check each of the cleaners for the valid files for (T cleaner : cleanersChain) { - if (cleaner.isStopped() || this.getStopper().isStopped()) { + if (cleaner.isStopped() || this.stopper.isStopped()) { LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any more files in:" + this.oldFileDir); return false; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java index 359315ef3bb3..ca2478c06bbf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java @@ -229,12 +229,9 @@ private void handleCreateTable(TableName tableName) ModifyRegionUtils.assignRegions(assignmentManager, regionInfos); } - // 8. Enable table + // 6. Enable table assignmentManager.getTableStateManager().setTableState(tableName, TableState.State.ENABLED); - - // 9. Update the tabledescriptor cache. - ((HMaster) this.server).getTableDescriptors().get(tableName); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java index e495e52c000c..8b1643819a5a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/StateDumpServlet.java @@ -38,10 +38,10 @@ public abstract class StateDumpServlet extends HttpServlet { protected void dumpVersionInfo(PrintWriter out) { VersionInfo.writeTo(out); - + out.println("Hadoop " + org.apache.hadoop.util.VersionInfo.getVersion()); - out.println("Source code repository " + org.apache.hadoop.util.VersionInfo.getUrl() - + " revision=" + org.apache.hadoop.util.VersionInfo.getRevision()); + out.println("Subversion " + org.apache.hadoop.util.VersionInfo.getUrl() + " -r " + + org.apache.hadoop.util.VersionInfo.getRevision()); out.println("Compiled by " + org.apache.hadoop.util.VersionInfo.getUser() + " on " + org.apache.hadoop.util.VersionInfo.getDate()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java index 2e5fc418bc75..d980c7c0c813 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ReplicationProtbufUtil.java @@ -138,7 +138,7 @@ public static void replicateWALEntry(final AdminService.BlockingInterface admin, List cells = edit.getCells(); // Add up the size. It is used later serializing out the kvs. for (Cell cell: cells) { - size += CellUtil.estimatedSerializedSizeOf(cell); + size += CellUtil.estimatedLengthOf(cell); } // Collect up the cells allCells.add(cells); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index c44a737eacbb..b67dc684cd68 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -209,8 +209,6 @@ public QuotaRefresherChore(final int period, final Stoppable stoppable) { } @Override - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="GC_UNRELATED_TYPES", - justification="I do not understand why the complaints, it looks good to me -- FIX") protected void chore() { // Prefetch online tables/namespaces for (TableName table: QuotaCache.this.rsServices.getOnlineTables()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java index 36e79291bba2..da3056c7f0c4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DelimitedKeyPrefixRegionSplitPolicy.java @@ -50,14 +50,20 @@ public class DelimitedKeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundR @Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); - // read the prefix length from the table descriptor - String delimiterString = region.getTableDesc().getValue(DELIMITER_KEY); - if (delimiterString == null || delimiterString.length() == 0) { - LOG.error(DELIMITER_KEY + " not specified for table " + region.getTableDesc().getTableName() + - ". Using default RegionSplitPolicy"); - return; + if (region != null) { + + // read the prefix length from the table descriptor + String delimiterString = region.getTableDesc().getValue( + DELIMITER_KEY); + if (delimiterString == null || delimiterString.length() == 0) { + LOG.error(DELIMITER_KEY + " not specified for table " + + region.getTableDesc().getTableName() + + ". Using default RegionSplitPolicy"); + return; + } + + delimiter = Bytes.toBytes(delimiterString); } - delimiter = Bytes.toBytes(delimiterString); } @Override @@ -79,4 +85,4 @@ protected byte[] getSplitPoint() { return splitPoint; } } -} \ No newline at end of file +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index fbf151af9b69..67ff2cb2d6b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5038,7 +5038,7 @@ public List get(Get get, boolean withCoprocessor) if (this.metricsRegion != null) { long totalSize = 0l; for (Cell cell : results) { - totalSize += CellUtil.estimatedSerializedSizeOf(cell); + totalSize += CellUtil.estimatedLengthOf(cell); } this.metricsRegion.updateGet(totalSize); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 45e5558497f4..a9aef78b6b70 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -517,7 +517,7 @@ public void uncaughtException(Thread t, Throwable e) { this.fs = new HFileSystem(this.conf, useHBaseChecksum); this.rootDir = FSUtils.getRootDir(this.conf); this.tableDescriptors = new FSTableDescriptors(this.conf, - this.fs, this.rootDir, !canUpdateTableDescriptor(), false); + this.fs, this.rootDir, !canUpdateTableDescriptor()); service = new ExecutorService(getServerName().toShortString()); spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); @@ -1357,8 +1357,6 @@ private RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLo (int) (store.getTotalStaticBloomSize() / 1024); } } - float dataLocality = - r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname()); if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } @@ -1381,8 +1379,7 @@ private RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLo .setWriteRequestsCount(r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) - .setCompleteSequenceId(r.lastFlushSeqId) - .setDataLocality(dataLocality); + .setCompleteSequenceId(r.lastFlushSeqId); return regionLoadBldr.build(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 8b41401b540c..8e7f576eda74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -44,6 +44,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -56,7 +57,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.conf.ConfigurationManager; import org.apache.hadoop.hbase.io.compress.Compression; @@ -91,6 +91,7 @@ import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; @@ -719,8 +720,8 @@ public void assertBulkLoadHFileOk(Path srcPath) throws IOException { if (CellComparator.compareRows(prevCell, cell) > 0) { throw new InvalidHFileException("Previous row is greater than" + " current row: path=" + srcPath + " previous=" - + CellUtil.getCellKeyAsString(prevCell) + " current=" - + CellUtil.getCellKeyAsString(cell)); + + CellUtil.getCellKey(prevCell) + " current=" + + CellUtil.getCellKey(cell)); } if (CellComparator.compareFamilies(prevCell, cell) != 0) { throw new InvalidHFileException("Previous key had different" @@ -976,7 +977,7 @@ private HFileContext createFileContext(Compression.Algorithm compression, .withIncludesMvcc(includeMVCCReadpoint) .withIncludesTags(includesTag) .withCompression(compression) - .withCompressTags(family.isCompressTags()) + .withCompressTags(family.shouldCompressTags()) .withChecksumType(checksumType) .withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java index b987a884cff4..dd6afd254634 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyPrefixRegionSplitPolicy.java @@ -43,34 +43,36 @@ public class KeyPrefixRegionSplitPolicy extends IncreasingToUpperBoundRegionSpli @Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); - prefixLength = 0; + if (region != null) { + prefixLength = 0; - // read the prefix length from the table descriptor - String prefixLengthString = region.getTableDesc().getValue( - PREFIX_LENGTH_KEY); - if (prefixLengthString == null) { - //read the deprecated value - prefixLengthString = region.getTableDesc().getValue(PREFIX_LENGTH_KEY_DEPRECATED); + // read the prefix length from the table descriptor + String prefixLengthString = region.getTableDesc().getValue( + PREFIX_LENGTH_KEY); if (prefixLengthString == null) { - LOG.error(PREFIX_LENGTH_KEY + " not specified for table " - + region.getTableDesc().getTableName() - + ". Using default RegionSplitPolicy"); + //read the deprecated value + prefixLengthString = region.getTableDesc().getValue(PREFIX_LENGTH_KEY_DEPRECATED); + if (prefixLengthString == null) { + LOG.error(PREFIX_LENGTH_KEY + " not specified for table " + + region.getTableDesc().getTableName() + + ". Using default RegionSplitPolicy"); + return; + } + } + try { + prefixLength = Integer.parseInt(prefixLengthString); + } catch (NumberFormatException nfe) { + /* Differentiate NumberFormatException from an invalid value range reported below. */ + LOG.error("Number format exception when parsing " + PREFIX_LENGTH_KEY + " for table " + + region.getTableDesc().getTableName() + ":" + + prefixLengthString + ". " + nfe); return; } - } - try { - prefixLength = Integer.parseInt(prefixLengthString); - } catch (NumberFormatException nfe) { - /* Differentiate NumberFormatException from an invalid value range reported below. */ - LOG.error("Number format exception when parsing " + PREFIX_LENGTH_KEY + " for table " - + region.getTableDesc().getTableName() + ":" - + prefixLengthString + ". " + nfe); - return; - } - if (prefixLength <= 0) { - LOG.error("Invalid value for " + PREFIX_LENGTH_KEY + " for table " - + region.getTableDesc().getTableName() + ":" - + prefixLengthString + ". Using default RegionSplitPolicy"); + if (prefixLength <= 0) { + LOG.error("Invalid value for " + PREFIX_LENGTH_KEY + " for table " + + region.getTableDesc().getTableName() + ":" + + prefixLengthString + ". Using default RegionSplitPolicy"); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 9eb2c0fa4481..21e4099e2fb0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -2071,7 +2071,7 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque for (Result r : results) { for (Cell cell : r.rawCells()) { currentScanResultSize += CellUtil.estimatedHeapSizeOf(cell); - totalCellSize += CellUtil.estimatedSerializedSizeOf(cell); + totalCellSize += CellUtil.estimatedLengthOf(cell); } } } @@ -2102,7 +2102,7 @@ public ScanResponse scan(final RpcController controller, final ScanRequest reque if (!values.isEmpty()) { for (Cell cell : values) { currentScanResultSize += CellUtil.estimatedHeapSizeOf(cell); - totalCellSize += CellUtil.estimatedSerializedSizeOf(cell); + totalCellSize += CellUtil.estimatedLengthOf(cell); } results.add(Result.create(values, null, stale)); i++; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index f02b8baa8fc0..67c499313c85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -139,12 +139,12 @@ void postOpenDeployTasks(final HRegion r) * @return all the online tables in this RS */ Set getOnlineTables(); - - + + /** * Registers a new protocol buffer {@link Service} subclass as a coprocessor endpoint to be * available for handling - * @param service the {@code Service} subclass instance to expose as a coprocessor endpoint + * @param instance the {@code Service} subclass instance to expose as a coprocessor endpoint * @return {@code true} if the registration was successful, {@code false} */ boolean registerService(Service service); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java index a8b314e9574f..1188ec803997 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; @@ -34,7 +33,7 @@ public class ScanInfo { private int minVersions; private int maxVersions; private long ttl; - private KeepDeletedCells keepDeletedCells; + private boolean keepDeletedCells; private long timeToPurgeDeletes; private KVComparator comparator; @@ -66,7 +65,7 @@ public ScanInfo(final HColumnDescriptor family, final long ttl, final long timeT * @param comparator The store's comparator */ public ScanInfo(final byte[] family, final int minVersions, final int maxVersions, - final long ttl, final KeepDeletedCells keepDeletedCells, final long timeToPurgeDeletes, + final long ttl, final boolean keepDeletedCells, final long timeToPurgeDeletes, final KVComparator comparator) { this.family = family; this.minVersions = minVersions; @@ -93,7 +92,7 @@ public long getTtl() { return ttl; } - public KeepDeletedCells getKeepDeletedCells() { + public boolean getKeepDeletedCells() { return keepDeletedCells; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 56e6b8a09f7c..964fad873e78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; @@ -73,7 +72,7 @@ public class ScanQueryMatcher { private boolean retainDeletesInOutput; /** whether to return deleted rows */ - private final KeepDeletedCells keepDeletedCells; + private final boolean keepDeletedCells; /** whether time range queries can see rows "behind" a delete */ private final boolean seePastDeleteMarkers; @@ -100,7 +99,6 @@ public class ScanQueryMatcher { * deleted KVs. */ private final long earliestPutTs; - private final long ttl; /** readPoint over which the KVs are unconditionally included */ protected long maxReadPointToTrackVersions; @@ -166,18 +164,15 @@ public ScanQueryMatcher(Scan scan, ScanInfo scanInfo, NavigableSet colum this.earliestPutTs = earliestPutTs; this.maxReadPointToTrackVersions = readPointToUse; this.timeToPurgeDeletes = scanInfo.getTimeToPurgeDeletes(); - this.ttl = oldestUnexpiredTS; /* how to deal with deletes */ this.isUserScan = scanType == ScanType.USER_SCAN; // keep deleted cells: if compaction or raw scan - this.keepDeletedCells = scan.isRaw() ? KeepDeletedCells.TRUE : - isUserScan ? KeepDeletedCells.FALSE : scanInfo.getKeepDeletedCells(); - // retain deletes: if minor compaction or raw scanisDone + this.keepDeletedCells = (scanInfo.getKeepDeletedCells() && !isUserScan) || scan.isRaw(); + // retain deletes: if minor compaction or raw scan this.retainDeletesInOutput = scanType == ScanType.COMPACT_RETAIN_DELETES || scan.isRaw(); // seePastDeleteMarker: user initiated scans - this.seePastDeleteMarkers = - scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE && isUserScan; + this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() && isUserScan; int maxVersions = scan.isRaw() ? scan.getMaxVersions() : Math.min(scan.getMaxVersions(), @@ -323,8 +318,7 @@ public MatchCode match(Cell cell) throws IOException { byte typeByte = cell.getTypeByte(); long mvccVersion = cell.getMvccVersion(); if (CellUtil.isDelete(cell)) { - if (keepDeletedCells == KeepDeletedCells.FALSE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < ttl)) { + if (!keepDeletedCells) { // first ignore delete markers if the scanner can do so, and the // range does not include the marker // @@ -354,8 +348,7 @@ public MatchCode match(Cell cell) throws IOException { // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking return MatchCode.INCLUDE; } - } else if (keepDeletedCells == KeepDeletedCells.TRUE - || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= ttl)) { + } else if (keepDeletedCells) { if (timestamp < earliestPutTs) { // keeping delete rows, but there are no puts older than // this delete in the store files. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 6d0098b9b2e8..0a4e1ed1a20f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -332,7 +332,7 @@ protected void seekScanners(List scanners, scanner.seek(seekKey); Cell c = scanner.peek(); if (c != null ) { - totalScannersSoughtBytes += CellUtil.estimatedSerializedSizeOf(c); + totalScannersSoughtBytes += CellUtil.estimatedSizeOf(c); } } } else { @@ -515,7 +515,7 @@ public boolean next(List outResult, int limit) throws IOException { if (this.countPerRow > storeOffset) { outResult.add(cell); count++; - totalBytesRead += CellUtil.estimatedSerializedSizeOf(cell); + totalBytesRead += CellUtil.estimatedSizeOf(cell); if (totalBytesRead > maxRowSize) { throw new RowTooBigException("Max row size allowed: " + maxRowSize + ", but the row is bigger than that."); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 2f820864e53b..42e4751f2a42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -1475,7 +1475,7 @@ public void postSync(final long timeInNanos, final int handlerSyncs) { public long postAppend(final Entry e, final long elapsedTime) { long len = 0; if (this.metrics == null) return len; - for (Cell cell : e.getEdit().getCells()) len += CellUtil.estimatedSerializedSizeOf(cell); + for (Cell cell : e.getEdit().getCells()) len += CellUtil.estimatedLengthOf(cell); metrics.finishAppend(elapsedTime, len); return len; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java index 1f81874357dd..aa0c3f379508 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -64,11 +64,11 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.TagRewriteCell; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.Delete; @@ -1974,10 +1974,7 @@ private static Cell tagReplayLogSequenceNumber(long seqId, Cell cell) { List newTags = new ArrayList(); Tag replayTag = new Tag(TagType.LOG_REPLAY_TAG_TYPE, Bytes.toBytes(seqId)); newTags.add(replayTag); - if (cell.getTagsLength() > 0) { - newTags.addAll(Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength())); - } - return new TagRewriteCell(cell, Tag.fromList(newTags)); + return KeyValue.cloneAndAddTags(cell, newTags); } return cell; } @@ -2048,7 +2045,9 @@ public static List getMutationsFromWALEntry(WALEntry entry, Cell } ((Put) m).add(tmpNewCell); } - m.setDurability(durability); + if (m != null) { + m.setDurability(durability); + } previousCell = cell; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index b186e089778c..2104268f2991 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -106,8 +106,8 @@ public ReplicationSinkManager(HConnection conn, String peerClusterId, * @return a replication sink to replicate to */ public SinkPeer getReplicationSink() throws IOException { - if (endpoint.getLastRegionServerUpdate() > this.lastUpdateToPeers || sinks.isEmpty()) { - LOG.info("Current list of sinks is out of date or empty, updating"); + if (endpoint.getLastRegionServerUpdate() > this.lastUpdateToPeers) { + LOG.info("Current list of sinks is out of date, updating"); chooseSinks(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 02b97f2309f0..04a2ad662375 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -161,10 +161,7 @@ public void init(final Configuration conf, final FileSystem fs, this.conf.getLong("replication.source.size.capacity", 1024*1024*64); this.replicationQueueNbCapacity = this.conf.getInt("replication.source.nb.capacity", 25000); - this.sleepForRetries = - this.conf.getLong("replication.source.sleepforretries", 1000); // 1 second - this.maxRetriesMultiplier = - this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per + this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 10); this.queue = new PriorityBlockingQueue( this.conf.getInt("hbase.regionserver.maxlogs", 32), @@ -174,6 +171,8 @@ public void init(final Configuration conf, final FileSystem fs, this.replicationQueues = replicationQueues; this.replicationPeers = replicationPeers; this.manager = manager; + this.sleepForRetries = + this.conf.getLong("replication.source.sleepforretries", 1000); this.fs = fs; this.metrics = metrics; this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java index ad1b088aefff..cb0f6ce68ff1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java @@ -137,8 +137,7 @@ public ReplicationSourceManager(final ReplicationQueues replicationQueues, this.fs = fs; this.logDir = logDir; this.oldLogDir = oldLogDir; - this.sleepBeforeFailover = - conf.getLong("replication.sleep.before.failover", 30000); // 30 seconds + this.sleepBeforeFailover = conf.getLong("replication.sleep.before.failover", 2000); this.clusterId = clusterId; this.replicationTracker.registerListener(this); this.replicationPeers.getAllPeerIds(); @@ -152,7 +151,6 @@ public ReplicationSourceManager(final ReplicationQueues replicationQueues, new LinkedBlockingQueue()); ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); tfb.setNameFormat("ReplicationExecutor-%d"); - tfb.setDaemon(true); this.executor.setThreadFactory(tfb.build()); this.rand = new Random(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java index 5ca1164980a6..007e610263b7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; +import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos; @@ -109,6 +110,20 @@ public class AccessControlLists { public static final char NAMESPACE_PREFIX = '@'; + /** Table descriptor for ACL internal table */ + public static final HTableDescriptor ACL_TABLEDESC = new HTableDescriptor(ACL_TABLE_NAME); + static { + ACL_TABLEDESC.addFamily( + new HColumnDescriptor(ACL_LIST_FAMILY, + 10, // Ten is arbitrary number. Keep versions to help debugging. + Compression.Algorithm.NONE.getName(), true, true, 8 * 1024, + HConstants.FOREVER, BloomType.NONE.toString(), + HConstants.REPLICATION_SCOPE_LOCAL). + // Set cache data blocks in L1 if more than one cache tier deployed; e.g. this will + // be the case if we are using CombinedBlockCache (Bucket Cache). + setCacheDataInL1(true)); + } + /** * Delimiter to separate user, column family, and qualifier in * _acl_ table info: column keys */ @@ -121,23 +136,11 @@ public class AccessControlLists { private static Log LOG = LogFactory.getLog(AccessControlLists.class); /** - * Create the ACL table - * @param master - * @throws IOException + * Check for existence of {@code _acl_} table and create it if it does not exist + * @param master reference to HMaster */ - static void createACLTable(MasterServices master) throws IOException { - master.createTable(new HTableDescriptor(ACL_TABLE_NAME) - .addFamily(new HColumnDescriptor(ACL_LIST_FAMILY) - .setMaxVersions(1) - .setInMemory(true) - .setBlockCacheEnabled(true) - .setBlocksize(8 * 1024) - .setBloomFilterType(BloomType.NONE) - .setScope(HConstants.REPLICATION_SCOPE_LOCAL) - // Set cache data blocks in L1 if more than one cache tier deployed; e.g. this will - // be the case if we are using CombinedBlockCache (Bucket Cache). - .setCacheDataInL1(true)), - null); + static void init(MasterServices master) throws IOException { + master.createTable(ACL_TABLEDESC, null); } /** @@ -461,7 +464,7 @@ static ListMultimap getNamespacePermissions(Configurati */ static ListMultimap getPermissions(Configuration conf, byte[] entryName) throws IOException { - if (entryName == null) entryName = ACL_GLOBAL_NAME; + if (entryName == null) entryName = ACL_TABLE_NAME.getName(); // for normal user tables, we just read the table row from _acl_ ListMultimap perms = ArrayListMultimap.create(); @@ -656,7 +659,7 @@ public static boolean isNamespaceEntry(String entryName) { public static boolean isNamespaceEntry(byte[] entryName) { return entryName[0] == NAMESPACE_PREFIX; } - + public static String toNamespaceEntry(String namespace) { return NAMESPACE_PREFIX + namespace; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 733f9dcda655..400e20a47835 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -49,8 +49,9 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.TagRewriteCell; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; @@ -62,7 +63,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseMasterAndRegionObserver; -import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorException; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.EndpointObserver; @@ -84,8 +84,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; -import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadRequest; -import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadRequest; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; @@ -152,7 +150,7 @@ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) public class AccessController extends BaseMasterAndRegionObserver implements RegionServerObserver, - AccessControlService.Interface, CoprocessorService, EndpointObserver, BulkLoadObserver { + AccessControlService.Interface, CoprocessorService, EndpointObserver { public static final Log LOG = LogFactory.getLog(AccessController.class); @@ -774,7 +772,13 @@ private static void addCellPermissions(final byte[] perms, Map ctx) if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getMasterServices() .getShortCircuitConnection(), AccessControlLists.ACL_TABLE_NAME)) { // initialize the ACL storage table - AccessControlLists.createACLTable(ctx.getEnvironment().getMasterServices()); + AccessControlLists.init(ctx.getEnvironment().getMasterServices()); } else { aclTabAvailable = true; } @@ -1296,7 +1300,7 @@ private void internalPreRead(final ObserverContext ourFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL, Lists.newArrayList(ourFilter, filter)); } - authResult.setAllowed(true); + authResult.setAllowed(true);; authResult.setReason("Access allowed with filter"); switch (opType) { case GET: @@ -1322,7 +1326,7 @@ private void internalPreRead(final ObserverContext ourFilter = new FilterList(FilterList.Operator.MUST_PASS_ALL, Lists.newArrayList(ourFilter, filter)); } - authResult.setAllowed(true); + authResult.setAllowed(true);; authResult.setReason("Access allowed with filter"); switch (opType) { case GET: @@ -1780,8 +1784,17 @@ public Cell postMutationBeforeWAL(ObserverContext return newCell; } - Cell rewriteCell = new TagRewriteCell(newCell, Tag.fromList(tags)); - return rewriteCell; + // We need to create another KV, unfortunately, because the current new KV + // has no space for tags + KeyValue rewriteKv = new KeyValue(newCell.getRowArray(), newCell.getRowOffset(), + newCell.getRowLength(), newCell.getFamilyArray(), newCell.getFamilyOffset(), + newCell.getFamilyLength(), newCell.getQualifierArray(), newCell.getQualifierOffset(), + newCell.getQualifierLength(), newCell.getTimestamp(), KeyValue.Type.codeToType(newCell + .getTypeByte()), newCell.getValueArray(), newCell.getValueOffset(), + newCell.getValueLength(), tags); + // Preserve mvcc data + rewriteKv.setSequenceId(newCell.getSequenceId()); + return rewriteKv; } @Override @@ -1878,15 +1891,11 @@ private AuthResult hasSomeAccess(RegionCoprocessorEnvironment e, String method, /** * Authorization check for * SecureBulkLoadProtocol.prepareBulkLoad() - * @param ctx the context - * @param request the request + * @param e * @throws IOException */ - @Override - public void prePrepareBulkLoad(ObserverContext ctx, - PrepareBulkLoadRequest request) throws IOException { - RegionCoprocessorEnvironment e = ctx.getEnvironment(); - + //TODO this should end up as a coprocessor hook + public void prePrepareBulkLoad(RegionCoprocessorEnvironment e) throws IOException { AuthResult authResult = hasSomeAccess(e, "prePrepareBulkLoad", Action.WRITE); logResult(authResult); if (!authResult.isAllowed()) { @@ -1898,15 +1907,11 @@ public void prePrepareBulkLoad(ObserverContext ctx /** * Authorization security check for * SecureBulkLoadProtocol.cleanupBulkLoad() - * @param ctx the context - * @param request the request + * @param e * @throws IOException */ - @Override - public void preCleanupBulkLoad(ObserverContext ctx, - CleanupBulkLoadRequest request) throws IOException { - RegionCoprocessorEnvironment e = ctx.getEnvironment(); - + //TODO this should end up as a coprocessor hook + public void preCleanupBulkLoad(RegionCoprocessorEnvironment e) throws IOException { AuthResult authResult = hasSomeAccess(e, "preCleanupBulkLoad", Action.WRITE); logResult(authResult); if (!authResult.isAllowed()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java index 764a12c09f1c..930d9b33ae10 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/SecureBulkLoadEndpoint.java @@ -34,9 +34,7 @@ import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.DoNotRetryIOException; -import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.ipc.RequestContext; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -159,18 +157,9 @@ public void prepareBulkLoad(RpcController controller, PrepareBulkLoadRequest request, RpcCallback done){ try { - List bulkLoadObservers = getBulkLoadObservers(); - - if(bulkLoadObservers != null) { - ObserverContext ctx = - new ObserverContext(); - ctx.prepare(env); - - for(BulkLoadObserver bulkLoadObserver : bulkLoadObservers) { - bulkLoadObserver.prePrepareBulkLoad(ctx, request); - } + if(userProvider.isHBaseSecurityEnabled()) { + getAccessController().prePrepareBulkLoad(env); } - String bulkToken = createStagingDir(baseStagingDir, getActiveUser(), ProtobufUtil.toTableName(request.getTableName())).toString(); done.run(PrepareBulkLoadResponse.newBuilder().setBulkToken(bulkToken).build()); @@ -185,18 +174,9 @@ public void cleanupBulkLoad(RpcController controller, CleanupBulkLoadRequest request, RpcCallback done) { try { - List bulkLoadObservers = getBulkLoadObservers(); - - if(bulkLoadObservers != null) { - ObserverContext ctx = - new ObserverContext(); - ctx.prepare(env); - - for(BulkLoadObserver bulkLoadObserver : bulkLoadObservers) { - bulkLoadObserver.preCleanupBulkLoad(ctx, request); - } + if (userProvider.isHBaseSecurityEnabled()) { + getAccessController().preCleanupBulkLoad(env); } - fs.delete(createStagingDir(baseStagingDir, getActiveUser(), new Path(request.getBulkToken()).getName()), @@ -312,11 +292,9 @@ public Boolean run() { done.run(SecureBulkLoadHFilesResponse.newBuilder().setLoaded(loaded).build()); } - private List getBulkLoadObservers() { - List coprocessorList = - this.env.getRegion().getCoprocessorHost().findCoprocessors(BulkLoadObserver.class); - - return coprocessorList; + private AccessController getAccessController() { + return (AccessController) this.env.getRegion() + .getCoprocessorHost().findCoprocessor(AccessController.class.getName()); } private Path createStagingDir(Path baseDir, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java index 0265e82dd4f1..c4c925ae52ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.List; -import java.util.concurrent.CountDownLatch; /** * Handles synchronization of access control list entries and updates @@ -49,7 +48,6 @@ public class ZKPermissionWatcher extends ZooKeeperListener { static final String ACL_NODE = "acl"; TableAuthManager authManager; String aclZNode; - CountDownLatch initialized = new CountDownLatch(1); public ZKPermissionWatcher(ZooKeeperWatcher watcher, TableAuthManager authManager, Configuration conf) { @@ -60,32 +58,18 @@ public ZKPermissionWatcher(ZooKeeperWatcher watcher, } public void start() throws KeeperException { - try { - watcher.registerListener(this); - if (ZKUtil.watchAndCheckExists(watcher, aclZNode)) { - List existing = - ZKUtil.getChildDataAndWatchForNewChildren(watcher, aclZNode); - if (existing != null) { - refreshNodes(existing); - } + watcher.registerListener(this); + if (ZKUtil.watchAndCheckExists(watcher, aclZNode)) { + List existing = + ZKUtil.getChildDataAndWatchForNewChildren(watcher, aclZNode); + if (existing != null) { + refreshNodes(existing); } - } finally { - initialized.countDown(); - } - } - - private void waitUntilStarted() { - try { - initialized.await(); - } catch (InterruptedException e) { - LOG.warn("Interrupted while waiting", e); - Thread.currentThread().interrupt(); } } @Override public void nodeCreated(String path) { - waitUntilStarted(); if (path.equals(aclZNode)) { try { List nodes = @@ -101,7 +85,6 @@ public void nodeCreated(String path) { @Override public void nodeDeleted(String path) { - waitUntilStarted(); if (aclZNode.equals(ZKUtil.getParent(path))) { String table = ZKUtil.getNodeName(path); if(AccessControlLists.isNamespaceEntry(table)) { @@ -114,7 +97,6 @@ public void nodeDeleted(String path) { @Override public void nodeDataChanged(String path) { - waitUntilStarted(); if (aclZNode.equals(ZKUtil.getParent(path))) { // update cache on an existing table node String entry = ZKUtil.getNodeName(path); @@ -133,7 +115,6 @@ public void nodeDataChanged(String path) { @Override public void nodeChildrenChanged(String path) { - waitUntilStarted(); if (path.equals(aclZNode)) { // table permissions changed try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index aaad8ba2574e..4986e0ffdb02 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -43,7 +43,8 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TagRewriteCell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.MetaTableAccessor; @@ -314,7 +315,12 @@ public void preBatchMutate(ObserverContext c, List tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); tags.addAll(visibilityTags); - Cell updatedCell = new TagRewriteCell(cell, Tag.fromList(tags)); + Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(), + cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), + cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell + .getTypeByte()), cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength(), tags); updatedCells.add(updatedCell); } m.getFamilyCellMap().clear(); @@ -324,6 +330,7 @@ public void preBatchMutate(ObserverContext c, Put p = (Put) m; p.add(cell); } else if (m instanceof Delete) { + // TODO : Cells without visibility tags would be handled in follow up issue Delete d = (Delete) m; d.addDeleteMarker(cell); } @@ -607,8 +614,17 @@ public Cell postMutationBeforeWAL(ObserverContext } } - Cell rewriteCell = new TagRewriteCell(newCell, Tag.fromList(tags)); - return rewriteCell; + // We need to create another KV, unfortunately, because the current new KV + // has no space for tags + KeyValue rewriteKv = new KeyValue(newCell.getRowArray(), newCell.getRowOffset(), + newCell.getRowLength(), newCell.getFamilyArray(), newCell.getFamilyOffset(), + newCell.getFamilyLength(), newCell.getQualifierArray(), newCell.getQualifierOffset(), + newCell.getQualifierLength(), newCell.getTimestamp(), KeyValue.Type.codeToType(newCell + .getTypeByte()), newCell.getValueArray(), newCell.getValueOffset(), + newCell.getValueLength(), tags); + // Preserve mvcc data + rewriteKv.setSequenceId(newCell.getSequenceId()); + return rewriteKv; } @Override @@ -724,7 +740,7 @@ public synchronized void getAuths(RpcController controller, GetAuthsRequest requ User requestingUser = VisibilityUtils.getActiveUser(); throw new AccessDeniedException("User '" + (requestingUser != null ? requestingUser.getShortName() : "null") - + "' is not authorized to perform this action."); + + " is not authorized to perform this action."); } labels = this.visibilityLabelService.getAuths(user, false); } catch (IOException e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java index 1323cdfbec27..96311d752634 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java @@ -94,7 +94,7 @@ public final int run(String[] args) throws IOException { cmd = parseArgs(args); cmdLineArgs = args; } catch (ParseException e) { - LOG.error("Error when parsing command-line arguments", e); + LOG.error("Error when parsing command-line arguemnts", e); printUsage(); return EXIT_FAILURE; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java index 56c37762f662..73b9edf65d18 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java @@ -415,11 +415,12 @@ public boolean contains(byte[] buf, int offset, int length, + " theBloom.limit()=" + theBloom.limit() + ", byteSize=" + byteSize); } - return contains(buf, offset, length, theBloom, 0, (int) byteSize, hash, hashCount); + return contains(buf, offset, length, theBloom.array(), + theBloom.arrayOffset(), (int) byteSize, hash, hashCount); } public static boolean contains(byte[] buf, int offset, int length, - ByteBuffer bloomBuf, int bloomOffset, int bloomSize, Hash hash, + byte[] bloomArray, int bloomOffset, int bloomSize, Hash hash, int hashCount) { int hash1 = hash.hash(buf, offset, length, 0); @@ -432,7 +433,7 @@ public static boolean contains(byte[] buf, int offset, int length, for (int i = 0; i < hashCount; i++) { int hashLoc = Math.abs(compositeHash % bloomBitSize); compositeHash += hash2; - if (!get(hashLoc, bloomBuf, bloomOffset)) { + if (!get(hashLoc, bloomArray, bloomOffset)) { return false; } } @@ -440,7 +441,7 @@ public static boolean contains(byte[] buf, int offset, int length, // Test mode with "fake lookups" to estimate "ideal false positive rate". for (int i = 0; i < hashCount; i++) { int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize); - if (!get(hashLoc, bloomBuf, bloomOffset)){ + if (!get(hashLoc, bloomArray, bloomOffset)){ return false; } } @@ -470,11 +471,10 @@ void set(long pos) { * @param pos index of bit * @return true if bit at specified index is 1, false if 0. */ - static boolean get(int pos, ByteBuffer bloomBuf, int bloomOffset) { + static boolean get(int pos, byte[] bloomArray, int bloomOffset) { int bytePos = pos >> 3; //pos / 8 int bitPos = pos & 0x7; //pos % 8 - // TODO access this via Util API which can do Unsafe access if possible(?) - byte curByte = bloomBuf.get(bloomOffset + bytePos); + byte curByte = bloomArray[bloomOffset + bytePos]; curByte &= bitvals[bitPos]; return (curByte != 0); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java index beda805ed1a5..2661f3f9c6ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java @@ -109,7 +109,7 @@ public boolean contains(byte[] key, int keyOffset, int keyLength, ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly(); result = ByteBloomFilter.contains(key, keyOffset, keyLength, - bloomBuf, bloomBlock.headerSize(), + bloomBuf.array(), bloomBuf.arrayOffset() + bloomBlock.headerSize(), bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java index abf3a0d0c371..850dc024d552 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java @@ -127,7 +127,7 @@ public static void doSmokeTest(FileSystem fs, Path path, String codec) // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); - writer.append(c); + writer.append(c);; writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java index bdf0f56f7b79..fb9059ea7c96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java @@ -59,7 +59,7 @@ * passed filesystem. It expects descriptors to be in a file in the * {@link #TABLEINFO_DIR} subdir of the table's directory in FS. Can be read-only * -- i.e. does not modify the filesystem or can be read and write. - * + * *

Also has utility for keeping up the table descriptors tableinfo file. * The table schema file is kept in the {@link #TABLEINFO_DIR} subdir * of the table directory in the filesystem. @@ -78,9 +78,6 @@ public class FSTableDescriptors implements TableDescriptors { private final FileSystem fs; private final Path rootdir; private final boolean fsreadonly; - private volatile boolean usecache; - private volatile boolean fsvisited; - @VisibleForTesting long cachehits = 0; @VisibleForTesting long invocations = 0; @@ -92,14 +89,43 @@ public class FSTableDescriptors implements TableDescriptors { // This cache does not age out the old stuff. Thinking is that the amount // of data we keep up in here is so small, no need to do occasional purge. // TODO. - private final Map cache = - new ConcurrentHashMap(); + private final Map cache = + new ConcurrentHashMap(); /** * Table descriptor for hbase:meta catalog table */ private final HTableDescriptor metaTableDescritor; + /** + * Data structure to hold modification time and table descriptor. + */ + private static class TableDescriptorAndModtime { + private final TableDescriptor td; + private final long modtime; + + TableDescriptorAndModtime(final long modtime, final TableDescriptor td) { + this.td = td; + this.modtime = modtime; + } + + long getModtime() { + return this.modtime; + } + + TableDescriptor getTableDescriptor() { + return this.td; + } + + HTableDescriptor getHTableDescriptor() { + return this.td.getHTableDescriptor(); + } + + TableState.State getTableState() { + return this.td.getTableState(); + } + } + /** * Construct a FSTableDescriptors instance using the hbase root dir of the given * conf and the filesystem where that root dir lives. @@ -108,10 +134,10 @@ public class FSTableDescriptors implements TableDescriptors { public FSTableDescriptors(final Configuration conf) throws IOException { this(conf, FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf)); } - + public FSTableDescriptors(final Configuration conf, final FileSystem fs, final Path rootdir) - throws IOException { - this(conf, fs, rootdir, false, true); + throws IOException { + this(conf, fs, rootdir, false); } /** @@ -119,34 +145,18 @@ public FSTableDescriptors(final Configuration conf, final FileSystem fs, final P * operations; i.e. on remove, we do not do delete in fs. */ public FSTableDescriptors(final Configuration conf, final FileSystem fs, - final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException { + final Path rootdir, final boolean fsreadonly) throws IOException { super(); this.fs = fs; this.rootdir = rootdir; this.fsreadonly = fsreadonly; - this.usecache = usecache; this.metaTableDescritor = TableDescriptor.metaTableDescriptor(conf); } - public void setCacheOn() throws IOException { - this.cache.clear(); - this.usecache = true; - } - - public void setCacheOff() throws IOException { - this.usecache = false; - this.cache.clear(); - } - - @VisibleForTesting - public boolean isUsecache() { - return this.usecache; - } - /** * Get the current table descriptor for the given table, or null if none exists. - * + * * Uses a local cache of the descriptor but still checks the filesystem on each call * to see if a newer file has been created since the cached one was read. */ @@ -165,17 +175,20 @@ public TableDescriptor getDescriptor(final TableName tablename) throw new IOException("No descriptor found for non table = " + tablename); } - if (usecache) { - // Look in cache of descriptors. - TableDescriptor cachedtdm = this.cache.get(tablename); - if (cachedtdm != null) { + // Look in cache of descriptors. + TableDescriptorAndModtime cachedtdm = this.cache.get(tablename); + + if (cachedtdm != null) { + // Check mod time has not changed (this is trip to NN). + if (getTableInfoModtime(tablename) <= cachedtdm.getModtime()) { cachehits++; - return cachedtdm; + return cachedtdm.getTableDescriptor(); } } - TableDescriptor tdmt = null; + + TableDescriptorAndModtime tdmt = null; try { - tdmt = getTableDescriptorFromFs(fs, rootdir, tablename, !fsreadonly); + tdmt = getTableDescriptorAndModtime(tablename); } catch (NullPointerException e) { LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, e); @@ -183,12 +196,11 @@ public TableDescriptor getDescriptor(final TableName tablename) LOG.debug("Exception during readTableDecriptor. Current table name = " + tablename, ioe); } - // last HTD written wins - if (usecache && tdmt != null) { + + if (tdmt != null) { this.cache.put(tablename, tdmt); } - - return tdmt; + return tdmt == null ? null : tdmt.getTableDescriptor(); } /** @@ -214,33 +226,17 @@ public HTableDescriptor get(TableName tableName) throws IOException { public Map getAllDescriptors() throws IOException { Map tds = new TreeMap(); - - if (fsvisited && usecache) { - for (Map.Entry entry: this.cache.entrySet()) { - tds.put(entry.getKey().toString(), entry.getValue()); - } - // add hbase:meta to the response - tds.put(this.metaTableDescritor.getNameAsString(), - new TableDescriptor(metaTableDescritor, TableState.State.ENABLED)); - } else { - LOG.debug("Fetching table descriptors from the filesystem."); - boolean allvisited = true; - for (Path d : FSUtils.getTableDirs(fs, rootdir)) { - TableDescriptor htd = null; - try { - htd = getDescriptor(FSUtils.getTableName(d)); - } catch (FileNotFoundException fnfe) { - // inability of retrieving one HTD shouldn't stop getting the remaining - LOG.warn("Trouble retrieving htd", fnfe); - } - if (htd == null) { - allvisited = false; - continue; - } else { - tds.put(htd.getHTableDescriptor().getTableName().getNameAsString(), htd); - } - fsvisited = allvisited; + List tableDirs = FSUtils.getTableDirs(fs, rootdir); + for (Path d: tableDirs) { + TableDescriptor htd = null; + try { + htd = getDescriptor(FSUtils.getTableName(d)); + } catch (FileNotFoundException fnfe) { + // inability of retrieving one HTD shouldn't stop getting the remaining + LOG.warn("Trouble retrieving htd", fnfe); } + if (htd == null) continue; + tds.put(htd.getHTableDescriptor().getTableName().getNameAsString(), htd); } return tds; } @@ -347,17 +343,13 @@ public HTableDescriptor remove(final TableName tablename) throw new IOException("Failed delete of " + tabledir.toString()); } } - TableDescriptor descriptor = this.cache.remove(tablename); - if (descriptor == null) { - return null; - } else { - return descriptor.getHTableDescriptor(); - } + TableDescriptorAndModtime tdm = this.cache.remove(tablename); + return tdm == null ? null : tdm.getHTableDescriptor(); } /** * Checks if a current table info file exists for the given table - * + * * @param tableName name of table * @return true if exists * @throws IOException @@ -365,7 +357,7 @@ public HTableDescriptor remove(final TableName tablename) public boolean isTableInfoExists(TableName tableName) throws IOException { return getTableInfoPath(tableName) != null; } - + /** * Find the most current table info file for the given table in the hbase root directory. * @return The file status of the current table info file or null if it does not exist @@ -379,15 +371,15 @@ private FileStatus getTableInfoPath(Path tableDir) throws IOException { return getTableInfoPath(fs, tableDir, !fsreadonly); } - + /** * Find the most current table info file for the table located in the given table directory. - * + * * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info * files and takes the 'current' one - meaning the one with the highest sequence number if present * or no sequence number at all if none exist (for backward compatibility from before there * were sequence numbers). - * + * * @return The file status of the current table info file or null if it does not exist * @throws IOException */ @@ -395,17 +387,17 @@ public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) throws IOException { return getTableInfoPath(fs, tableDir, false); } - + /** * Find the most current table info file for the table in the given table directory. - * + * * Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info * files and takes the 'current' one - meaning the one with the highest sequence number if * present or no sequence number at all if none exist (for backward compatibility from before * there were sequence numbers). * If there are multiple table info files found and removeOldFiles is true it also deletes the * older files. - * + * * @return The file status of the current table info file or null if none exist * @throws IOException */ @@ -414,17 +406,17 @@ private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles); } - + /** * Find the most current table info file in the given directory - * + * * Looks within the given directory for any table info files * and takes the 'current' one - meaning the one with the highest sequence number if present * or no sequence number at all if none exist (for backward compatibility from before there * were sequence numbers). * If there are multiple possible files found * and the we're not in read only mode it also deletes the older files. - * + * * @return The file status of the current table info file or null if it does not exist * @throws IOException */ @@ -454,7 +446,7 @@ static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir, boolean rem } return mostCurrent; } - + /** * Compare {@link FileStatus} instances by {@link Path#getName()}. Returns in * reverse order. @@ -479,7 +471,7 @@ public int compare(FileStatus left, FileStatus right) { public boolean accept(Path p) { // Accept any file that starts with TABLEINFO_NAME return p.getName().startsWith(TABLEINFO_FILE_PREFIX); - }}; + }}; /** * Width of the sequenceid that is a suffix on a tableinfo file. @@ -523,6 +515,7 @@ private static String formatTableInfoSequenceId(final int number) { } /** + * @param tabledir * @param sequenceid * @return Name of tableinfo file. */ @@ -531,14 +524,16 @@ private static String formatTableInfoSequenceId(final int number) { } /** - * Returns the latest table descriptor for the given table directly from the file system - * if it exists, bypassing the local cache. - * Returns null if it's not found. + * @param fs + * @param rootdir + * @param tableName + * @return Modification time for the table {@link #TABLEINFO_FILE_PREFIX} file + * or 0 if no tableinfo file found. + * @throws IOException */ - public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, - Path hbaseRootDir, TableName tableName) throws IOException { - Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName); - return getTableDescriptorFromFs(fs, tableDir); + private long getTableInfoModtime(final TableName tableName) throws IOException { + FileStatus status = getTableInfoPath(tableName); + return status == null ? 0 : status.getModificationTime(); } /** @@ -547,33 +542,53 @@ public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, * Returns null if it's not found. */ public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, - Path hbaseRootDir, TableName tableName, boolean rewritePb) throws IOException { + Path hbaseRootDir, TableName tableName) throws IOException { Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName); - return getTableDescriptorFromFs(fs, tableDir, rewritePb); + return getTableDescriptorFromFs(fs, tableDir); } + /** * Returns the latest table descriptor for the table located at the given directory * directly from the file system if it exists. * @throws TableInfoMissingException if there is no descriptor */ public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir) - throws IOException { - return getTableDescriptorFromFs(fs, tableDir, false); + throws IOException { + FileStatus status = getTableInfoPath(fs, tableDir, false); + if (status == null) { + throw new TableInfoMissingException("No table descriptor file under " + tableDir); + } + return readTableDescriptor(fs, status, false); + } + + /** + * @param tableName table name + * @return TableDescriptorAndModtime or null if no table descriptor was found + * @throws IOException + */ + private TableDescriptorAndModtime getTableDescriptorAndModtime(TableName tableName) + throws IOException { + // ignore both -ROOT- and hbase:meta tables + if (tableName.equals(TableName.META_TABLE_NAME)) { + return null; + } + return getTableDescriptorAndModtime(getTableDir(tableName)); } /** - * Returns the latest table descriptor for the table located at the given directory - * directly from the file system if it exists. - * @throws TableInfoMissingException if there is no descriptor + * @param tableDir path to table directory + * @return TableDescriptorAndModtime or null if no table descriptor was found + * at the specified path + * @throws IOException */ - public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir, - boolean rewritePb) + private TableDescriptorAndModtime getTableDescriptorAndModtime(Path tableDir) throws IOException { - FileStatus status = getTableInfoPath(fs, tableDir, false); + FileStatus status = getTableInfoPath(tableDir); if (status == null) { - throw new TableInfoMissingException("No table descriptor file under " + tableDir); + return null; } - return readTableDescriptor(fs, status, rewritePb); + TableDescriptor td = readTableDescriptor(fs, status, !fsreadonly); + return new TableDescriptorAndModtime(status.getModificationTime(), td); } private static TableDescriptor readTableDescriptor(FileSystem fs, FileStatus status, @@ -631,9 +646,8 @@ private static void rewriteTableDescriptor(final FileSystem fs, final FileStatus Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(tableDir)); if (p == null) throw new IOException("Failed update"); LOG.info("Updated tableinfo=" + p); - if (usecache) { - this.cache.put(td.getHTableDescriptor().getTableName(), td); - } + long modtime = getTableInfoModtime(tableName); + this.cache.put(tableName, new TableDescriptorAndModtime(modtime, td)); return p; } @@ -646,14 +660,14 @@ public void deleteTableDescriptorIfExists(TableName tableName) throws IOExceptio if (fsreadonly) { throw new NotImplementedException("Cannot delete a table descriptor - in read only mode"); } - + Path tableDir = getTableDir(tableName); Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); deleteTableDescriptorFiles(fs, tableInfoDir, Integer.MAX_VALUE); } /** - * Deletes files matching the table info file pattern within the given directory + * Deletes files matching the table info file pattern within the given directory * whose sequenceId is at most the given max sequenceId. */ private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId) @@ -672,25 +686,25 @@ private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxS } } } - + /** * Attempts to write a new table descriptor to the given table's directory. * It first writes it to the .tmp dir then uses an atomic rename to move it into place. * It begins at the currentSequenceId + 1 and tries 10 times to find a new sequence number * not already in use. * Removes the current descriptor file if passed in. - * + * * @return Descriptor file or null if we failed write. */ - private static Path writeTableDescriptor(final FileSystem fs, + private static Path writeTableDescriptor(final FileSystem fs, final TableDescriptor htd, final Path tableDir, final FileStatus currentDescriptorFile) - throws IOException { + throws IOException { // Get temporary dir into which we'll first write a file to avoid half-written file phenomenon. // This directory is never removed to avoid removing it out from under a concurrent writer. Path tmpTableDir = new Path(tableDir, TMP_DIR); Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); - + // What is current sequenceid? We read the current sequenceid from // the current file. After we read it, another thread could come in and // compete with us writing out next version of file. The below retries @@ -699,7 +713,7 @@ private static Path writeTableDescriptor(final FileSystem fs, int currentSequenceId = currentDescriptorFile == null ? 0 : getTableInfoSequenceId(currentDescriptorFile.getPath()); int newSequenceId = currentSequenceId; - + // Put arbitrary upperbound on how often we retry int retries = 10; int retrymax = currentSequenceId + retries; @@ -737,7 +751,7 @@ private static Path writeTableDescriptor(final FileSystem fs, } return tableInfoDirPath; } - + private static void writeTD(final FileSystem fs, final Path p, final TableDescriptor htd) throws IOException { FSDataOutputStream out = fs.create(p, false); @@ -772,7 +786,7 @@ public boolean createTableDescriptor(HTableDescriptor htd) throws IOException { * Create new HTableDescriptor in HDFS. Happens when we are creating table. If * forceCreation is true then even if previous table descriptor is present it * will be overwritten - * + * * @return True if we successfully created file. */ public boolean createTableDescriptor(TableDescriptor htd, boolean forceCreation) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 8dad6922c28e..284d4aba2924 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -656,11 +656,11 @@ public static void setVersion(FileSystem fs, Path rootdir, String version, if (s != null) s.close(); } catch (IOException ignore) { } } - LOG.info("Created version file at " + rootdir.toString() + " with version=" + version); + LOG.debug("Created version file at " + rootdir.toString() + " with version=" + version); return; } catch (IOException e) { if (retries > 0) { - LOG.debug("Unable to create version file at " + rootdir.toString() + ", retrying", e); + LOG.warn("Unable to create version file at " + rootdir.toString() + ", retrying", e); fs.delete(versionFile, false); try { if (wait > 0) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 9373130c9839..b06e70a3782d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -118,6 +119,7 @@ import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; @@ -226,7 +228,6 @@ public class HBaseFsck extends Configured { private boolean fixReferenceFiles = false; // fix lingering reference store file private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows private boolean fixTableLocks = false; // fix table locks which are expired - private boolean fixAny = false; // Set to true if any of the fix is required. // limit checking/fixes to listed tables, if empty attempt to check/fix all // hbase:meta are always checked @@ -379,7 +380,6 @@ public void connect() throws IOException { // kill the hbck with a ctrl-c, we want to cleanup the lock so that // it is available for further calls Runtime.getRuntime().addShutdownHook(new Thread() { - @Override public void run() { unlockHbck(); } @@ -1133,7 +1133,7 @@ public void fixOrphanTables() throws IOException { FSTableDescriptors fstd = new FSTableDescriptors(getConf()); while (iter.hasNext()) { Entry> entry = - iter.next(); + (Entry>) iter.next(); TableName tableName = entry.getKey(); LOG.info("Trying to fix orphan table error: " + tableName); if (j < htds.length) { @@ -3401,7 +3401,7 @@ private void printTableSummary(SortedMap tablesInfo) { static ErrorReporter getErrorReporter( final Configuration conf) throws ClassNotFoundException { Class reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class); - return ReflectionUtils.newInstance(reporter, conf); + return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf); } public interface ErrorReporter { @@ -3743,7 +3743,7 @@ public synchronized Void call() throws IOException { * Display the full report from fsck. This displays all live and dead region * servers, and all known regions. */ - public void setDisplayFullReport() { + public static void setDisplayFullReport() { details = true; } @@ -3776,7 +3776,6 @@ void setRegionBoundariesCheck() { */ public void setFixTableLocks(boolean shouldFix) { fixTableLocks = shouldFix; - fixAny |= shouldFix; } /** @@ -3799,7 +3798,6 @@ boolean shouldRerun() { */ public void setFixAssignments(boolean shouldFix) { fixAssignments = shouldFix; - fixAny |= shouldFix; } boolean shouldFixAssignments() { @@ -3808,7 +3806,6 @@ boolean shouldFixAssignments() { public void setFixMeta(boolean shouldFix) { fixMeta = shouldFix; - fixAny |= shouldFix; } boolean shouldFixMeta() { @@ -3817,7 +3814,6 @@ boolean shouldFixMeta() { public void setFixEmptyMetaCells(boolean shouldFix) { fixEmptyMetaCells = shouldFix; - fixAny |= shouldFix; } boolean shouldFixEmptyMetaCells() { @@ -3834,7 +3830,6 @@ boolean shouldCheckHdfs() { public void setFixHdfsHoles(boolean shouldFix) { fixHdfsHoles = shouldFix; - fixAny |= shouldFix; } boolean shouldFixHdfsHoles() { @@ -3843,7 +3838,6 @@ boolean shouldFixHdfsHoles() { public void setFixTableOrphans(boolean shouldFix) { fixTableOrphans = shouldFix; - fixAny |= shouldFix; } boolean shouldFixTableOrphans() { @@ -3852,7 +3846,6 @@ boolean shouldFixTableOrphans() { public void setFixHdfsOverlaps(boolean shouldFix) { fixHdfsOverlaps = shouldFix; - fixAny |= shouldFix; } boolean shouldFixHdfsOverlaps() { @@ -3861,7 +3854,6 @@ boolean shouldFixHdfsOverlaps() { public void setFixHdfsOrphans(boolean shouldFix) { fixHdfsOrphans = shouldFix; - fixAny |= shouldFix; } boolean shouldFixHdfsOrphans() { @@ -3870,7 +3862,6 @@ boolean shouldFixHdfsOrphans() { public void setFixVersionFile(boolean shouldFix) { fixVersionFile = shouldFix; - fixAny |= shouldFix; } public boolean shouldFixVersionFile() { @@ -3887,7 +3878,6 @@ public boolean shouldSidelineBigOverlaps() { public void setFixSplitParents(boolean shouldFix) { fixSplitParents = shouldFix; - fixAny |= shouldFix; } boolean shouldFixSplitParents() { @@ -3896,7 +3886,6 @@ boolean shouldFixSplitParents() { public void setFixReferenceFiles(boolean shouldFix) { fixReferenceFiles = shouldFix; - fixAny |= shouldFix; } boolean shouldFixReferenceFiles() { @@ -3904,7 +3893,7 @@ boolean shouldFixReferenceFiles() { } public boolean shouldIgnorePreCheckPermission() { - return !fixAny || ignorePreCheckPermission; + return ignorePreCheckPermission; } public void setIgnorePreCheckPermission(boolean ignorePreCheckPermission) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java index 81678aa36f0b..1dc9c310d454 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java @@ -77,18 +77,20 @@ public MultiHConnection(Configuration conf, int noOfConnections) public void close() { synchronized (hConnectionsLock) { if (hConnections != null) { - for (Connection conn : hConnections) { - if (conn != null) { - try { - conn.close(); - } catch (IOException e) { - LOG.info("Got exception in closing connection", e); - } finally { - conn = null; + if (hConnections != null) { + for (Connection conn : hConnections) { + if (conn != null) { + try { + conn.close(); + } catch (IOException e) { + LOG.info("Got exception in closing connection", e); + } finally { + conn = null; + } } } + hConnections = null; } - hConnections = null; } } if (this.batchPool != null && !this.batchPool.isShutdown()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java deleted file mode 100644 index 120f1707de5a..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/YammerHistogramUtils.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.stats.Sample; -import com.yammer.metrics.stats.Snapshot; - -import java.lang.reflect.Constructor; -import java.text.DecimalFormat; - -/** Utility functions for working with Yammer Metrics. */ -public final class YammerHistogramUtils { - - // not for public consumption - private YammerHistogramUtils() {} - - /** - * Used formatting doubles so only two places after decimal point. - */ - private static DecimalFormat DOUBLE_FORMAT = new DecimalFormat("#0.00"); - - /** - * Create a new {@link com.yammer.metrics.core.Histogram} instance. These constructors are - * not public in 2.2.0, so we use reflection to find them. - */ - public static Histogram newHistogram(Sample sample) { - try { - Constructor ctor = - Histogram.class.getDeclaredConstructor(Sample.class); - ctor.setAccessible(true); - return (Histogram) ctor.newInstance(sample); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** @return an abbreviated summary of {@code hist}. */ - public static String getShortHistogramReport(final Histogram hist) { - Snapshot sn = hist.getSnapshot(); - return "mean=" + DOUBLE_FORMAT.format(hist.mean()) + - ", min=" + DOUBLE_FORMAT.format(hist.min()) + - ", max=" + DOUBLE_FORMAT.format(hist.max()) + - ", stdDev=" + DOUBLE_FORMAT.format(hist.stdDev()) + - ", 95th=" + DOUBLE_FORMAT.format(sn.get95thPercentile()) + - ", 99th=" + DOUBLE_FORMAT.format(sn.get99thPercentile()); - } - - /** @return a summary of {@code hist}. */ - public static String getHistogramReport(final Histogram hist) { - Snapshot sn = hist.getSnapshot(); - return ", mean=" + DOUBLE_FORMAT.format(hist.mean()) + - ", min=" + DOUBLE_FORMAT.format(hist.min()) + - ", max=" + DOUBLE_FORMAT.format(hist.max()) + - ", stdDev=" + DOUBLE_FORMAT.format(hist.stdDev()) + - ", 50th=" + DOUBLE_FORMAT.format(sn.getMedian()) + - ", 75th=" + DOUBLE_FORMAT.format(sn.get75thPercentile()) + - ", 95th=" + DOUBLE_FORMAT.format(sn.get95thPercentile()) + - ", 99th=" + DOUBLE_FORMAT.format(sn.get99thPercentile()) + - ", 99.9th=" + DOUBLE_FORMAT.format(sn.get999thPercentile()) + - ", 99.99th=" + DOUBLE_FORMAT.format(sn.getValue(0.9999)) + - ", 99.999th=" + DOUBLE_FORMAT.format(sn.getValue(0.99999)); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java index e3ac47e1d97e..8d8d2108875c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java @@ -165,7 +165,8 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException, standaloneServerFactory = new NIOServerCnxnFactory(); standaloneServerFactory.configure( new InetSocketAddress(tentativePort), - configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, 1000)); + configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, + 1000)); } catch (BindException e) { LOG.debug("Failed binding ZK Server to client port: " + tentativePort, e); @@ -180,7 +181,6 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException, // Start up this ZK server standaloneServerFactory.startup(server); - // Runs a 'stat' against the servers. if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) { throw new IOException("Waiting for startup of standalone server"); } @@ -196,8 +196,8 @@ public int startup(File baseDir, int numZooKeeperServers) throws IOException, activeZKServerIndex = 0; started = true; int clientPort = clientPortList.get(activeZKServerIndex); - LOG.info("Started MiniZooKeeperCluster and ran successful 'stat' " + - "on client port=" + clientPort); + LOG.info("Started MiniZK Cluster and connect 1 ZK server " + + "on client port: " + clientPort); return clientPort; } diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 2b7829ca77c4..519f88310f07 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -48,10 +48,10 @@ String tableHeader; boolean withReplica = false; if (table.getTableDescriptor().getRegionReplication() > 1) { - tableHeader = "

Table Regions

"; + tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyLocalityRequestsReplicaID
"; withReplica = true; } else { - tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyRequestsReplicaID
"; + tableHeader = "

Table Regions

NameRegion ServerStart KeyEnd KeyLocalityRequests
"; } ServerName rl = metaTableLocator.getMetaRegionLocation(master.getZooKeeper()); boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); @@ -212,10 +212,9 @@ + - - <% } %>
NameRegion ServerStart KeyEnd KeyRequests
<%= escapeXml(meta.getRegionNameAsString()) %> <%= metaLocation.getHostname().toString() + ":" + master.getRegionServerInfoPort(metaLocation) %>- <%= escapeXml(Bytes.toString(meta.getStartKey())) %> <%= escapeXml(Bytes.toString(meta.getEndKey())) %>--
@@ -269,7 +268,7 @@ HRegionInfo regionInfo = hriEntry.getKey(); ServerName addr = hriEntry.getValue(); long req = 0; - float locality = 0.0f; + String urlRegionServer = null; if (addr != null) { @@ -278,7 +277,6 @@ Map map = sl.getRegionsLoad(); if (map.containsKey(regionInfo.getRegionName())) { req = map.get(regionInfo.getRegionName()).getRequestsCount(); - locality = map.get(regionInfo.getRegionName()).getDataLocality(); } Integer i = regDistribution.get(addr); if (null == i) i = Integer.valueOf(0); @@ -307,7 +305,6 @@ conf))) %> <%= escapeXml(Bytes.toStringBinary(HRegionInfo.getEndKeyForDisplay(regionInfo, conf))) %> - <%= locality%> <%= req%> <% if (withReplica) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java index 76a9566d8084..586552053f22 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java @@ -95,7 +95,7 @@ public ClusterStatus getInitialClusterStatus() throws IOException { } /** - * Returns an {@link MasterService.BlockingInterface} to the active master + * Returns an {@link MasterAdminService.BlockingInterface} to the active master */ public abstract MasterService.BlockingInterface getMasterAdminService() throws IOException; @@ -150,8 +150,7 @@ public void waitForRegionServerToStart(String hostname, long timeout) } Threads.sleep(100); } - throw new IOException("did timeout " + timeout + "ms waiting for region server to start: " - + hostname); + throw new IOException("did timeout waiting for region server to start:" + hostname); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index d0553fc5b976..18bc731fe6f9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -52,7 +52,6 @@ * like an HBaseConfiguration and filesystem. * @deprecated Write junit4 unit tests using {@link HBaseTestingUtility} */ -@Deprecated public abstract class HBaseTestCase extends TestCase { private static final Log LOG = LogFactory.getLog(HBaseTestCase.class); @@ -112,12 +111,12 @@ protected void setUp() throws Exception { } try { if (localfs) { - testDir = getUnitTestdir(getName()); + this.testDir = getUnitTestdir(getName()); if (fs.exists(testDir)) { fs.delete(testDir, true); } } else { - testDir = FSUtils.getRootDir(conf); + this.testDir = FSUtils.getRootDir(conf); } } catch (Exception e) { LOG.fatal("error during setup", e); @@ -207,7 +206,7 @@ protected HTableDescriptor createTableDescriptor(final String name, * @return Column descriptor. */ protected HTableDescriptor createTableDescriptor(final String name, - final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { + final int minVersions, final int versions, final int ttl, boolean keepDeleted) { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { htd.addFamily(new HColumnDescriptor(cfName) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 0c12b551ff85..450e805028ef 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1599,7 +1599,7 @@ public void deleteTableIfAny(TableName tableName) throws IOException { * @return Column descriptor. */ public HTableDescriptor createTableDescriptor(final String name, - final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) { + final int minVersions, final int versions, final int ttl, boolean keepDeleted) { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) { htd.addFamily(new HColumnDescriptor(cfName) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 08b09465f4a8..f54776577348 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -74,7 +74,10 @@ import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.trace.SpanReceiverHost; -import org.apache.hadoop.hbase.util.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Hash; +import org.apache.hadoop.hbase.util.MurmurHash; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; @@ -176,25 +179,6 @@ protected static enum Counter { ROWS } - protected static class RunResult implements Comparable { - public RunResult(long duration, Histogram hist) { - this.duration = duration; - this.hist = hist; - } - - public final long duration; - public final Histogram hist; - - @Override - public String toString() { - return Long.toString(duration); - } - - @Override public int compareTo(RunResult o) { - return Long.compare(this.duration, o.duration); - } - } - /** * Constructor * @param conf Configuration object @@ -274,12 +258,12 @@ public void setStatus(String msg) { final Connection con = ConnectionFactory.createConnection(conf); // Evaluation task - RunResult result = PerformanceEvaluation.runOneClient(this.cmd, conf, con, opts, status); + long elapsedTime = runOneClient(this.cmd, conf, con, opts, status); // Collect how much time the thing took. Report as map output and // to the ELAPSED_TIME counter. - context.getCounter(Counter.ELAPSED_TIME).increment(result.duration); + context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime); context.getCounter(Counter.ROWS).increment(opts.perClientRunRows); - context.write(new LongWritable(opts.startRow), new LongWritable(result.duration)); + context.write(new LongWritable(opts.startRow), new LongWritable(elapsedTime)); context.progress(); } } @@ -384,32 +368,34 @@ protected static byte[][] getSplits(TestOptions opts) { /* * Run all clients in this vm each to its own thread. + * @param cmd Command to run. + * @throws IOException */ - static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf) + static long doLocalClients(final TestOptions opts, final Configuration conf) throws IOException, InterruptedException { final Class cmd = determineCommandClass(opts.cmdName); assert cmd != null; - Future[] threads = new Future[opts.numClientThreads]; - RunResult[] results = new RunResult[opts.numClientThreads]; + Future[] threads = new Future[opts.numClientThreads]; + long[] timings = new long[opts.numClientThreads]; ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads, new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build()); final Connection con = ConnectionFactory.createConnection(conf); for (int i = 0; i < threads.length; i++) { final int index = i; - threads[i] = pool.submit(new Callable() { + threads[i] = pool.submit(new Callable() { @Override - public RunResult call() throws Exception { + public Long call() throws Exception { TestOptions threadOpts = new TestOptions(opts); if (threadOpts.startRow == 0) threadOpts.startRow = index * threadOpts.perClientRunRows; - RunResult run = runOneClient(cmd, conf, con, threadOpts, new Status() { + long elapsedTime = runOneClient(cmd, conf, con, threadOpts, new Status() { @Override public void setStatus(final String msg) throws IOException { LOG.info(msg); } }); - LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration + + LOG.info("Finished in " + elapsedTime + "ms over " + threadOpts.perClientRunRows + " rows"); - return run; + return elapsedTime; } }); } @@ -417,27 +403,27 @@ public void setStatus(final String msg) throws IOException { for (int i = 0; i < threads.length; i++) { try { - results[i] = threads[i].get(); + timings[i] = threads[i].get(); } catch (ExecutionException e) { throw new IOException(e.getCause()); } } final String test = cmd.getSimpleName(); LOG.info("[" + test + "] Summary of timings (ms): " - + Arrays.toString(results)); - Arrays.sort(results); + + Arrays.toString(timings)); + Arrays.sort(timings); long total = 0; - for (RunResult result : results) { - total += result.duration; + for (long timing : timings) { + total += timing; } LOG.info("[" + test + "]" - + "\tMin: " + results[0] + "ms" - + "\tMax: " + results[results.length - 1] + "ms" - + "\tAvg: " + (total / results.length) + "ms"); + + "\tMin: " + timings[0] + "ms" + + "\tMax: " + timings[timings.length - 1] + "ms" + + "\tAvg: " + (total / timings.length) + "ms"); con.close(); - return results; + return total; } /* @@ -1007,21 +993,23 @@ protected int getReportingPeriod() { return opts.period; } - /** - * Populated by testTakedown. Only implemented by RandomReadTest at the moment. - */ - public Histogram getLatency() { - return latency; - } - void testSetup() throws IOException { if (!opts.oneCon) { this.connection = ConnectionFactory.createConnection(conf); } this.table = new HTable(TableName.valueOf(opts.tableName), connection); this.table.setAutoFlushTo(opts.autoFlush); - latency = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); - valueSize = YammerHistogramUtils.newHistogram(new UniformSample(1024 * 500)); + + try { + Constructor ctor = + Histogram.class.getDeclaredConstructor(com.yammer.metrics.stats.Sample.class); + ctor.setAccessible(true); + latency = (Histogram) ctor.newInstance(new UniformSample(1024 * 500)); + valueSize = (Histogram) ctor.newInstance(new UniformSample(1024 * 500)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } void testTakedown() throws IOException { @@ -1102,7 +1090,6 @@ private void reportHistogram(final Histogram h) throws IOException { status.setStatus(testName + " Avg = " + h.mean()); status.setStatus(testName + " StdDev = " + h.stdDev()); status.setStatus(testName + " 50th = " + sn.getMedian()); - status.setStatus(testName + " 75th = " + sn.get75thPercentile()); status.setStatus(testName + " 95th = " + sn.get95thPercentile()); status.setStatus(testName + " 99th = " + sn.get99thPercentile()); status.setStatus(testName + " 99.9th = " + sn.get999thPercentile()); @@ -1111,18 +1098,33 @@ private void reportHistogram(final Histogram h) throws IOException { status.setStatus(testName + " Max = " + h.max()); } + /** + * Used formating doubles so only two places after decimal point. + */ + private static DecimalFormat DOUBLE_FORMAT = new DecimalFormat("#0.00"); + /** * @return Subset of the histograms' calculation. */ - public String getShortLatencyReport() { - return YammerHistogramUtils.getShortHistogramReport(this.latency); + private String getShortLatencyReport() { + return getShortHistogramReport(this.latency); } /** * @return Subset of the histograms' calculation. */ - public String getShortValueSizeReport() { - return YammerHistogramUtils.getShortHistogramReport(this.valueSize); + private String getShortValueSizeReport() { + return getShortHistogramReport(this.valueSize); + } + + private String getShortHistogramReport(final Histogram h) { + Snapshot sn = h.getSnapshot(); + return "mean=" + DOUBLE_FORMAT.format(h.mean()) + + ", min=" + DOUBLE_FORMAT.format(h.min()) + + ", max=" + DOUBLE_FORMAT.format(h.max()) + + ", stdDev=" + DOUBLE_FORMAT.format(h.stdDev()) + + ", 95th=" + DOUBLE_FORMAT.format(sn.get95thPercentile()) + + ", 99th=" + DOUBLE_FORMAT.format(sn.get99thPercentile()); } /* @@ -1526,7 +1528,7 @@ public static byte[] generateValue(final Random r) { return format(random.nextInt(Integer.MAX_VALUE) % totalRows); } - static RunResult runOneClient(final Class cmd, Configuration conf, Connection con, + static long runOneClient(final Class cmd, Configuration conf, Connection con, TestOptions opts, final Status status) throws IOException, InterruptedException { status.setStatus("Start " + cmd + " at offset " + opts.startRow + " for " + @@ -1553,7 +1555,7 @@ static RunResult runOneClient(final Class cmd, Configuration con " (" + calculateMbps((int)(opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts)) + ")"); - return new RunResult(totalElapsedTime, t.getLatency()); + return totalElapsedTime; } private static int getAverageValueLength(final TestOptions opts) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index e19b8fef2770..2718daa743f1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -82,15 +82,12 @@ public class TestIOFencing { static final Log LOG = LogFactory.getLog(TestIOFencing.class); static { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) - // .getLogger().setLevel(Level.ALL); - //((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")).getLogger().setLevel(Level.ALL); + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); } public abstract static class CompactionBlockerRegion extends HRegion { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java index f3bf9c4842cc..aeb86467b7b6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java @@ -131,9 +131,6 @@ public static ClusterConnection getMockedConnectionAndDecorate(final Configurati RpcControllerFactory.instantiate(conf))); Mockito.doNothing().when(c).incCount(); Mockito.doNothing().when(c).decCount(); - Mockito.when(c.getNewRpcRetryingCallerFactory(conf)).thenReturn( - RpcRetryingCallerFactory.instantiate(conf, - RetryingCallerInterceptorFactory.NO_OP_INTERCEPTOR)); return c; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java similarity index 62% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 3d3b499e640e..edb1f6f33c47 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -30,39 +30,60 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.InvalidFamilyOperationException; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import com.google.protobuf.ServiceException; + + /** * Class to test HBaseAdmin. * Spins up the minicluster once at test start and then takes it down afterward. * Add any testing of HBaseAdmin functionality here. */ @Category({LargeTests.class, ClientTests.class}) -public class TestAdmin1 { +public class TestAdmin { final Log LOG = LogFactory.getLog(getClass()); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private Admin admin; @@ -1110,4 +1131,632 @@ public void testEnableDisableAddColumnDeleteColumn() throws Exception { this.admin.disableTable(tableName); this.admin.deleteTable(tableName); } + + @Test (timeout=300000) + public void testCreateBadTables() throws IOException { + String msg = null; + try { + this.admin.createTable(new HTableDescriptor(TableName.META_TABLE_NAME)); + } catch(TableExistsException e) { + msg = e.toString(); + } + assertTrue("Unexcepted exception message " + msg, msg != null && + msg.startsWith(TableExistsException.class.getName()) && + msg.contains(TableName.META_TABLE_NAME.getNameAsString())); + + // Now try and do concurrent creation with a bunch of threads. + final HTableDescriptor threadDesc = + new HTableDescriptor(TableName.valueOf("threaded_testCreateBadTables")); + threadDesc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + int count = 10; + Thread [] threads = new Thread [count]; + final AtomicInteger successes = new AtomicInteger(0); + final AtomicInteger failures = new AtomicInteger(0); + final Admin localAdmin = this.admin; + for (int i = 0; i < count; i++) { + threads[i] = new Thread(Integer.toString(i)) { + @Override + public void run() { + try { + localAdmin.createTable(threadDesc); + successes.incrementAndGet(); + } catch (TableExistsException e) { + failures.incrementAndGet(); + } catch (IOException e) { + throw new RuntimeException("Failed threaded create" + getName(), e); + } + } + }; + } + for (int i = 0; i < count; i++) { + threads[i].start(); + } + for (int i = 0; i < count; i++) { + while(threads[i].isAlive()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // continue + } + } + } + // All threads are now dead. Count up how many tables were created and + // how many failed w/ appropriate exception. + assertEquals(1, successes.get()); + assertEquals(count - 1, failures.get()); + } + + /** + * Test for hadoop-1581 'HBASE: Unopenable tablename bug'. + * @throws Exception + */ + @Test (timeout=300000) + public void testTableNameClash() throws Exception { + String name = "testTableNameClash"; + HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf(name + "SOMEUPPERCASE")); + HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf(name)); + htd1.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + htd2.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(htd1); + admin.createTable(htd2); + // Before fix, below would fail throwing a NoServerForRegionException. + new HTable(TEST_UTIL.getConfiguration(), htd2.getTableName()).close(); + } + + /*** + * HMaster.createTable used to be kind of synchronous call + * Thus creating of table with lots of regions can cause RPC timeout + * After the fix to make createTable truly async, RPC timeout shouldn't be an + * issue anymore + * @throws Exception + */ + @Test (timeout=300000) + public void testCreateTableRPCTimeOut() throws Exception { + String name = "testCreateTableRPCTimeOut"; + int oldTimeout = TEST_UTIL.getConfiguration(). + getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); + try { + int expectedRegions = 100; + // Use 80 bit numbers to make sure we aren't limited + byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; + Admin hbaseadmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); + htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + hbaseadmin.createTable(htd, startKey, endKey, expectedRegions); + hbaseadmin.close(); + } finally { + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout); + } + } + + /** + * Test read only tables + * @throws Exception + */ + @Test (timeout=300000) + public void testReadOnlyTable() throws Exception { + TableName name = TableName.valueOf("testReadOnlyTable"); + Table table = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + byte[] value = Bytes.toBytes("somedata"); + // This used to use an empty row... That must have been a bug + Put put = new Put(value); + put.add(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, value); + table.put(put); + table.close(); + } + + /** + * Test that user table names can contain '-' and '.' so long as they do not + * start with same. HBASE-771 + * @throws IOException + */ + @Test (timeout=300000) + public void testTableNames() throws IOException { + byte[][] illegalNames = new byte[][] { + Bytes.toBytes("-bad"), + Bytes.toBytes(".bad") + }; + for (byte[] illegalName : illegalNames) { + try { + new HTableDescriptor(TableName.valueOf(illegalName)); + throw new IOException("Did not detect '" + + Bytes.toString(illegalName) + "' as an illegal user table name"); + } catch (IllegalArgumentException e) { + // expected + } + } + byte[] legalName = Bytes.toBytes("g-oo.d"); + try { + new HTableDescriptor(TableName.valueOf(legalName)); + } catch (IllegalArgumentException e) { + throw new IOException("Legal user table name: '" + + Bytes.toString(legalName) + "' caused IllegalArgumentException: " + + e.getMessage()); + } + } + + /** + * For HADOOP-2579 + * @throws IOException + */ + @Test (expected=TableExistsException.class, timeout=300000) + public void testTableExistsExceptionWithATable() throws IOException { + final TableName name = TableName.valueOf("testTableExistsExceptionWithATable"); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + } + + /** + * Can't disable a table if the table isn't in enabled state + * @throws IOException + */ + @Test (expected=TableNotEnabledException.class, timeout=300000) + public void testTableNotEnabledExceptionWithATable() throws IOException { + final TableName name = TableName.valueOf("testTableNotEnabledExceptionWithATable"); + TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); + this.admin.disableTable(name); + this.admin.disableTable(name); + } + + /** + * Can't enable a table if the table isn't in disabled state + * @throws IOException + */ + @Test (expected=TableNotDisabledException.class, timeout=300000) + public void testTableNotDisabledExceptionWithATable() throws IOException { + final TableName name = TableName.valueOf("testTableNotDisabledExceptionWithATable"); + Table t = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); + try { + this.admin.enableTable(name); + }finally { + t.close(); + } + } + + /** + * For HADOOP-2579 + * @throws IOException + */ + @Test (expected=TableNotFoundException.class, timeout=300000) + public void testTableNotFoundExceptionWithoutAnyTables() throws IOException { + TableName tableName = TableName + .valueOf("testTableNotFoundExceptionWithoutAnyTables"); + Table ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + ht.get(new Get("e".getBytes())); + } + + + @Test (timeout=300000) + public void testShouldCloseTheRegionBasedOnTheEncodedRegionName() + throws Exception { + TableName TABLENAME = + TableName.valueOf("TestHBACloseRegion"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.getTable().isSystemTable()) { + info = regionInfo; + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), rs + .getServerName().getServerName()); + } + } + boolean isInList = ProtobufUtil.getOnlineRegions( + rs.getRSRpcServices()).contains(info); + long timeout = System.currentTimeMillis() + 10000; + while ((System.currentTimeMillis() < timeout) && (isInList)) { + Thread.sleep(100); + isInList = ProtobufUtil.getOnlineRegions( + rs.getRSRpcServices()).contains(info); + } + + assertFalse("The region should not be present in online regions list.", + isInList); + } + + @Test (timeout=300000) + public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion1"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion1")) { + info = regionInfo; + try { + admin.closeRegionWithEncodedRegionName("sample", rs.getServerName() + .getServerName()); + } catch (NotServingRegionException nsre) { + // expected, ignore it + } + } + } + } + onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + assertTrue("The region should be present in online regions list.", + onlineRegions.contains(info)); + } + + @Test (timeout=300000) + public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception { + TableName TABLENAME = + TableName.valueOf("TestHBACloseRegion2"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) { + info = regionInfo; + admin.closeRegion(regionInfo.getRegionNameAsString(), rs + .getServerName().getServerName()); + } + } + } + + boolean isInList = ProtobufUtil.getOnlineRegions( + rs.getRSRpcServices()).contains(info); + long timeout = System.currentTimeMillis() + 10000; + while ((System.currentTimeMillis() < timeout) && (isInList)) { + Thread.sleep(100); + isInList = ProtobufUtil.getOnlineRegions( + rs.getRSRpcServices()).contains(info); + } + + assertFalse("The region should not be present in online regions list.", + isInList); + } + + @Test (timeout=300000) + public void testCloseRegionWhenServerNameIsNull() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion3"); + createTableWithDefaultConf(TABLENAME); + + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); + + try { + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString() + .contains("TestHBACloseRegion3")) { + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), + null); + } + } + } + fail("The test should throw exception if the servername passed is null."); + } catch (IllegalArgumentException e) { + } + } + + + @Test (timeout=300000) + public void testCloseRegionWhenServerNameIsEmpty() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); + createTableWithDefaultConf(TABLENAME); + + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); + + try { + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString() + .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { + admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), + " "); + } + } + } + fail("The test should throw exception if the servername passed is empty."); + } catch (IllegalArgumentException e) { + } + } + + @Test (timeout=300000) + public void testCloseRegionWhenEncodedRegionNameIsNotGiven() throws Exception { + byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion4"); + createTableWithDefaultConf(TABLENAME); + + HRegionInfo info = null; + HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); + + List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + for (HRegionInfo regionInfo : onlineRegions) { + if (!regionInfo.isMetaTable()) { + if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion4")) { + info = regionInfo; + try { + admin.closeRegionWithEncodedRegionName(regionInfo + .getRegionNameAsString(), rs.getServerName().getServerName()); + } catch (NotServingRegionException nsre) { + // expected, ignore it. + } + } + } + } + onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); + assertTrue("The region should be present in online regions list.", + onlineRegions.contains(info)); + } + + private HBaseAdmin createTable(byte[] TABLENAME) throws IOException { + + Configuration config = TEST_UTIL.getConfiguration(); + HBaseAdmin admin = new HBaseAdmin(config); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLENAME)); + HColumnDescriptor hcd = new HColumnDescriptor("value"); + + htd.addFamily(hcd); + admin.createTable(htd, null); + return admin; + } + + private void createTableWithDefaultConf(byte[] TABLENAME) throws IOException { + createTableWithDefaultConf(TableName.valueOf(TABLENAME)); + } + + private void createTableWithDefaultConf(TableName TABLENAME) throws IOException { + HTableDescriptor htd = new HTableDescriptor(TABLENAME); + HColumnDescriptor hcd = new HColumnDescriptor("value"); + htd.addFamily(hcd); + + admin.createTable(htd, null); + } + + /** + * For HBASE-2556 + * @throws IOException + */ + @Test (timeout=300000) + public void testGetTableRegions() throws IOException { + + final TableName tableName = TableName.valueOf("testGetTableRegions"); + + int expectedRegions = 10; + + // Use 80 bit numbers to make sure we aren't limited + byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; + byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; + + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, startKey, endKey, expectedRegions); + + List RegionInfos = admin.getTableRegions(tableName); + + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + RegionInfos.size(), + expectedRegions, RegionInfos.size()); + + } + + @Test (timeout=300000) + public void testHLogRollWriting() throws Exception { + setUpforLogRolling(); + String className = this.getClass().getName(); + StringBuilder v = new StringBuilder(className); + while (v.length() < 1000) { + v.append(className); + } + byte[] value = Bytes.toBytes(v.toString()); + HRegionServer regionServer = startAndWriteData(TableName.valueOf("TestLogRolling"), value); + LOG.info("after writing there are " + + HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()) + " log files"); + + // flush all regions + + List regions = new ArrayList(regionServer + .getOnlineRegionsLocalContext()); + for (HRegion r : regions) { + r.flushcache(); + } + admin.rollHLogWriter(regionServer.getServerName().getServerName()); + int count = HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()); + LOG.info("after flushing all regions and rolling logs there are " + + count + " log files"); + assertTrue(("actual count: " + count), count <= 2); + } + + @Test (timeout=300000) + public void testMoveToPreviouslyAssignedRS() throws IOException, InterruptedException { + byte[] tableName = Bytes.toBytes("testMoveToPreviouslyAssignedRS"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); + HBaseAdmin localAdmin = createTable(tableName); + List tableRegions = localAdmin.getTableRegions(tableName); + HRegionInfo hri = tableRegions.get(0); + AssignmentManager am = master.getAssignmentManager(); + assertTrue("Region " + hri.getRegionNameAsString() + + " should be assigned properly", am.waitForAssignment(hri)); + ServerName server = am.getRegionStates().getRegionServerOfRegion(hri); + localAdmin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(server.getServerName())); + assertEquals("Current region server and region server before move should be same.", server, + am.getRegionStates().getRegionServerOfRegion(hri)); + } + + + private void setUpforLogRolling() { + // Force a region split after every 768KB + TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, + 768L * 1024L); + + // We roll the log after every 32 writes + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32); + + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.logroll.errors.tolerated", 2); + TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000); + + // For less frequently updated regions flush after every 2 flushes + TEST_UTIL.getConfiguration().setInt( + "hbase.hregion.memstore.optionalflushcount", 2); + + // We flush the cache after every 8192 bytes + TEST_UTIL.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, + 8192); + + // Increase the amount of time between client retries + TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000); + + // Reduce thread wake frequency so that other threads can get + // a chance to run. + TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY, + 2 * 1000); + + /**** configuration for testLogRollOnDatanodeDeath ****/ + // make sure log.hflush() calls syncFs() to open a pipeline + TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); + // lower the namenode & datanode heartbeat so the namenode + // quickly detects datanode failures + TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000); + TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); + // the namenode might still try to choose the recently-dead datanode + // for a pipeline, so try to a new pipeline multiple times + TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.tolerable.lowreplication", 2); + TEST_UTIL.getConfiguration().setInt( + "hbase.regionserver.hlog.lowreplication.rolllimit", 3); + } + + private HRegionServer startAndWriteData(TableName tableName, byte[] value) + throws IOException, InterruptedException { + // When the hbase:meta table can be opened, the region servers are running + new HTable( + TEST_UTIL.getConfiguration(), TableName.META_TABLE_NAME).close(); + + // Create the test table and open it + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc); + Table table = new HTable(TEST_UTIL.getConfiguration(), tableName); + + HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName); + for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls + Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i))); + put.add(HConstants.CATALOG_FAMILY, null, value); + table.put(put); + if (i % 32 == 0) { + // After every 32 writes sleep to let the log roller run + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + // continue + } + } + } + + table.close(); + return regionServer; + } + + /** + * HBASE-4417 checkHBaseAvailable() doesn't close zk connections + */ + @Test (timeout=300000) + public void testCheckHBaseAvailableClosesConnection() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + + int initialCount = HConnectionTestingUtility.getConnectionCount(); + HBaseAdmin.checkHBaseAvailable(conf); + int finalCount = HConnectionTestingUtility.getConnectionCount(); + + Assert.assertEquals(initialCount, finalCount) ; + } + + /** + * Check that we have an exception if the cluster is not there. + */ + @Test (timeout=300000) + public void testCheckHBaseAvailableWithoutCluster() { + Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); + + // Change the ZK address to go to something not used. + conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, + conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 9999)+10); + + int initialCount = HConnectionTestingUtility.getConnectionCount(); + + long start = System.currentTimeMillis(); + try { + HBaseAdmin.checkHBaseAvailable(conf); + assertTrue(false); + } catch (MasterNotRunningException ignored) { + } catch (ZooKeeperConnectionException ignored) { + } catch (ServiceException ignored) { + } catch (IOException ignored) { + } + long end = System.currentTimeMillis(); + + int finalCount = HConnectionTestingUtility.getConnectionCount(); + + Assert.assertEquals(initialCount, finalCount) ; + + LOG.info("It took "+(end-start)+" ms to find out that" + + " HBase was not available"); + } + + @Test (timeout=300000) + public void testDisableCatalogTable() throws Exception { + try { + this.admin.disableTable(TableName.META_TABLE_NAME); + fail("Expected to throw ConstraintException"); + } catch (ConstraintException e) { + } + // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table + // actually getting disabled by the disableTable() call. + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testDisableCatalogTable".getBytes())); + HColumnDescriptor hcd = new HColumnDescriptor("cf1".getBytes()); + htd.addFamily(hcd); + TEST_UTIL.getHBaseAdmin().createTable(htd); + } + + @Test (timeout=300000) + public void testIsEnabledOrDisabledOnUnknownTable() throws Exception { + try { + admin.isTableEnabled(TableName.valueOf("unkownTable")); + fail("Test should fail if isTableEnabled called on unknown table."); + } catch (IOException e) { + } + + try { + admin.isTableDisabled(TableName.valueOf("unkownTable")); + fail("Test should fail if isTableDisabled called on unknown table."); + } catch (IOException e) { + } + } + + @Test (timeout=300000) + public void testGetRegion() throws Exception { + // We use actual HBaseAdmin instance instead of going via Admin interface in + // here because makes use of an internal HBA method (TODO: Fix.). + HBaseAdmin rawAdmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); + + final TableName tableName = TableName.valueOf("testGetRegion"); + LOG.info("Started " + tableName); + HTable t = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); + TEST_UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); + + HRegionLocation regionLocation = t.getRegionLocation("mmm"); + HRegionInfo region = regionLocation.getRegionInfo(); + byte[] regionName = region.getRegionName(); + Pair pair = rawAdmin.getRegion(regionName); + assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); + pair = rawAdmin.getRegion(region.getEncodedNameAsBytes()); + assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java deleted file mode 100644 index 396f8ea43a14..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java +++ /dev/null @@ -1,734 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.NotServingRegionException; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotDisabledException; -import org.apache.hadoop.hbase.TableNotEnabledException; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.wal.HLogUtilsForTests; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.google.protobuf.ServiceException; - - -/** - * Class to test HBaseAdmin. - * Spins up the minicluster once at test start and then takes it down afterward. - * Add any testing of HBaseAdmin functionality here. - */ -@Category({LargeTests.class, ClientTests.class}) -public class TestAdmin2 { - final Log LOG = LogFactory.getLog(getClass()); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private Admin admin; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); - TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); - TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); - TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); - TEST_UTIL.getConfiguration().setBoolean( - "hbase.master.enabletable.roundrobin", true); - TEST_UTIL.startMiniCluster(3); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getHBaseAdmin(); - } - - @After - public void tearDown() throws Exception { - for (HTableDescriptor htd : this.admin.listTables()) { - TEST_UTIL.deleteTable(htd.getName()); - } - } - - @Test (timeout=300000) - public void testCreateBadTables() throws IOException { - String msg = null; - try { - this.admin.createTable(new HTableDescriptor(TableName.META_TABLE_NAME)); - } catch(TableExistsException e) { - msg = e.toString(); - } - assertTrue("Unexcepted exception message " + msg, msg != null && - msg.startsWith(TableExistsException.class.getName()) && - msg.contains(TableName.META_TABLE_NAME.getNameAsString())); - - // Now try and do concurrent creation with a bunch of threads. - final HTableDescriptor threadDesc = - new HTableDescriptor(TableName.valueOf("threaded_testCreateBadTables")); - threadDesc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - int count = 10; - Thread [] threads = new Thread [count]; - final AtomicInteger successes = new AtomicInteger(0); - final AtomicInteger failures = new AtomicInteger(0); - final Admin localAdmin = this.admin; - for (int i = 0; i < count; i++) { - threads[i] = new Thread(Integer.toString(i)) { - @Override - public void run() { - try { - localAdmin.createTable(threadDesc); - successes.incrementAndGet(); - } catch (TableExistsException e) { - failures.incrementAndGet(); - } catch (IOException e) { - throw new RuntimeException("Failed threaded create" + getName(), e); - } - } - }; - } - for (int i = 0; i < count; i++) { - threads[i].start(); - } - for (int i = 0; i < count; i++) { - while(threads[i].isAlive()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - // continue - } - } - } - // All threads are now dead. Count up how many tables were created and - // how many failed w/ appropriate exception. - assertEquals(1, successes.get()); - assertEquals(count - 1, failures.get()); - } - - /** - * Test for hadoop-1581 'HBASE: Unopenable tablename bug'. - * @throws Exception - */ - @Test (timeout=300000) - public void testTableNameClash() throws Exception { - String name = "testTableNameClash"; - HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf(name + "SOMEUPPERCASE")); - HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf(name)); - htd1.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - htd2.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin.createTable(htd1); - admin.createTable(htd2); - // Before fix, below would fail throwing a NoServerForRegionException. - new HTable(TEST_UTIL.getConfiguration(), htd2.getTableName()).close(); - } - - /*** - * HMaster.createTable used to be kind of synchronous call - * Thus creating of table with lots of regions can cause RPC timeout - * After the fix to make createTable truly async, RPC timeout shouldn't be an - * issue anymore - * @throws Exception - */ - @Test (timeout=300000) - public void testCreateTableRPCTimeOut() throws Exception { - String name = "testCreateTableRPCTimeOut"; - int oldTimeout = TEST_UTIL.getConfiguration(). - getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); - TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500); - try { - int expectedRegions = 100; - // Use 80 bit numbers to make sure we aren't limited - byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; - Admin hbaseadmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); - htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - hbaseadmin.createTable(htd, startKey, endKey, expectedRegions); - hbaseadmin.close(); - } finally { - TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, oldTimeout); - } - } - - /** - * Test read only tables - * @throws Exception - */ - @Test (timeout=300000) - public void testReadOnlyTable() throws Exception { - TableName name = TableName.valueOf("testReadOnlyTable"); - Table table = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); - byte[] value = Bytes.toBytes("somedata"); - // This used to use an empty row... That must have been a bug - Put put = new Put(value); - put.add(HConstants.CATALOG_FAMILY, HConstants.CATALOG_FAMILY, value); - table.put(put); - table.close(); - } - - /** - * Test that user table names can contain '-' and '.' so long as they do not - * start with same. HBASE-771 - * @throws IOException - */ - @Test (timeout=300000) - public void testTableNames() throws IOException { - byte[][] illegalNames = new byte[][] { - Bytes.toBytes("-bad"), - Bytes.toBytes(".bad") - }; - for (byte[] illegalName : illegalNames) { - try { - new HTableDescriptor(TableName.valueOf(illegalName)); - throw new IOException("Did not detect '" + - Bytes.toString(illegalName) + "' as an illegal user table name"); - } catch (IllegalArgumentException e) { - // expected - } - } - byte[] legalName = Bytes.toBytes("g-oo.d"); - try { - new HTableDescriptor(TableName.valueOf(legalName)); - } catch (IllegalArgumentException e) { - throw new IOException("Legal user table name: '" + - Bytes.toString(legalName) + "' caused IllegalArgumentException: " + - e.getMessage()); - } - } - - /** - * For HADOOP-2579 - * @throws IOException - */ - @Test (expected=TableExistsException.class, timeout=300000) - public void testTableExistsExceptionWithATable() throws IOException { - final TableName name = TableName.valueOf("testTableExistsExceptionWithATable"); - TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); - TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); - } - - /** - * Can't disable a table if the table isn't in enabled state - * @throws IOException - */ - @Test (expected=TableNotEnabledException.class, timeout=300000) - public void testTableNotEnabledExceptionWithATable() throws IOException { - final TableName name = TableName.valueOf("testTableNotEnabledExceptionWithATable"); - TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY).close(); - this.admin.disableTable(name); - this.admin.disableTable(name); - } - - /** - * Can't enable a table if the table isn't in disabled state - * @throws IOException - */ - @Test (expected=TableNotDisabledException.class, timeout=300000) - public void testTableNotDisabledExceptionWithATable() throws IOException { - final TableName name = TableName.valueOf("testTableNotDisabledExceptionWithATable"); - Table t = TEST_UTIL.createTable(name, HConstants.CATALOG_FAMILY); - try { - this.admin.enableTable(name); - }finally { - t.close(); - } - } - - /** - * For HADOOP-2579 - * @throws IOException - */ - @Test (expected=TableNotFoundException.class, timeout=300000) - public void testTableNotFoundExceptionWithoutAnyTables() throws IOException { - TableName tableName = TableName - .valueOf("testTableNotFoundExceptionWithoutAnyTables"); - Table ht = new HTable(TEST_UTIL.getConfiguration(), tableName); - ht.get(new Get("e".getBytes())); - } - - @Test (timeout=300000) - public void testShouldCloseTheRegionBasedOnTheEncodedRegionName() - throws Exception { - TableName TABLENAME = - TableName.valueOf("TestHBACloseRegion"); - createTableWithDefaultConf(TABLENAME); - - HRegionInfo info = null; - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.getTable().isSystemTable()) { - info = regionInfo; - admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), rs - .getServerName().getServerName()); - } - } - boolean isInList = ProtobufUtil.getOnlineRegions( - rs.getRSRpcServices()).contains(info); - long timeout = System.currentTimeMillis() + 10000; - while ((System.currentTimeMillis() < timeout) && (isInList)) { - Thread.sleep(100); - isInList = ProtobufUtil.getOnlineRegions( - rs.getRSRpcServices()).contains(info); - } - - assertFalse("The region should not be present in online regions list.", - isInList); - } - - @Test (timeout=300000) - public void testCloseRegionIfInvalidRegionNameIsPassed() throws Exception { - byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion1"); - createTableWithDefaultConf(TABLENAME); - - HRegionInfo info = null; - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.isMetaTable()) { - if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion1")) { - info = regionInfo; - try { - admin.closeRegionWithEncodedRegionName("sample", rs.getServerName() - .getServerName()); - } catch (NotServingRegionException nsre) { - // expected, ignore it - } - } - } - } - onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - assertTrue("The region should be present in online regions list.", - onlineRegions.contains(info)); - } - - @Test (timeout=300000) - public void testCloseRegionThatFetchesTheHRIFromMeta() throws Exception { - TableName TABLENAME = - TableName.valueOf("TestHBACloseRegion2"); - createTableWithDefaultConf(TABLENAME); - - HRegionInfo info = null; - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLENAME); - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.isMetaTable()) { - - if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion2")) { - info = regionInfo; - admin.closeRegion(regionInfo.getRegionNameAsString(), rs - .getServerName().getServerName()); - } - } - } - - boolean isInList = ProtobufUtil.getOnlineRegions( - rs.getRSRpcServices()).contains(info); - long timeout = System.currentTimeMillis() + 10000; - while ((System.currentTimeMillis() < timeout) && (isInList)) { - Thread.sleep(100); - isInList = ProtobufUtil.getOnlineRegions( - rs.getRSRpcServices()).contains(info); - } - - assertFalse("The region should not be present in online regions list.", - isInList); - } - - @Test (timeout=300000) - public void testCloseRegionWhenServerNameIsNull() throws Exception { - byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion3"); - createTableWithDefaultConf(TABLENAME); - - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); - - try { - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.isMetaTable()) { - if (regionInfo.getRegionNameAsString() - .contains("TestHBACloseRegion3")) { - admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), - null); - } - } - } - fail("The test should throw exception if the servername passed is null."); - } catch (IllegalArgumentException e) { - } - } - - @Test (timeout=300000) - public void testCloseRegionWhenServerNameIsEmpty() throws Exception { - byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegionWhenServerNameIsEmpty"); - createTableWithDefaultConf(TABLENAME); - - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); - - try { - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.isMetaTable()) { - if (regionInfo.getRegionNameAsString() - .contains("TestHBACloseRegionWhenServerNameIsEmpty")) { - admin.closeRegionWithEncodedRegionName(regionInfo.getEncodedName(), - " "); - } - } - } - fail("The test should throw exception if the servername passed is empty."); - } catch (IllegalArgumentException e) { - } - } - - @Test (timeout=300000) - public void testCloseRegionWhenEncodedRegionNameIsNotGiven() throws Exception { - byte[] TABLENAME = Bytes.toBytes("TestHBACloseRegion4"); - createTableWithDefaultConf(TABLENAME); - - HRegionInfo info = null; - HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TableName.valueOf(TABLENAME)); - - List onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - for (HRegionInfo regionInfo : onlineRegions) { - if (!regionInfo.isMetaTable()) { - if (regionInfo.getRegionNameAsString().contains("TestHBACloseRegion4")) { - info = regionInfo; - try { - admin.closeRegionWithEncodedRegionName(regionInfo - .getRegionNameAsString(), rs.getServerName().getServerName()); - } catch (NotServingRegionException nsre) { - // expected, ignore it. - } - } - } - } - onlineRegions = ProtobufUtil.getOnlineRegions(rs.getRSRpcServices()); - assertTrue("The region should be present in online regions list.", - onlineRegions.contains(info)); - } - - private HBaseAdmin createTable(byte[] TABLENAME) throws IOException { - - Configuration config = TEST_UTIL.getConfiguration(); - HBaseAdmin admin = new HBaseAdmin(config); - - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLENAME)); - HColumnDescriptor hcd = new HColumnDescriptor("value"); - - htd.addFamily(hcd); - admin.createTable(htd, null); - return admin; - } - - private void createTableWithDefaultConf(byte[] TABLENAME) throws IOException { - createTableWithDefaultConf(TableName.valueOf(TABLENAME)); - } - - private void createTableWithDefaultConf(TableName TABLENAME) throws IOException { - HTableDescriptor htd = new HTableDescriptor(TABLENAME); - HColumnDescriptor hcd = new HColumnDescriptor("value"); - htd.addFamily(hcd); - - admin.createTable(htd, null); - } - - /** - * For HBASE-2556 - * @throws IOException - */ - @Test (timeout=300000) - public void testGetTableRegions() throws IOException { - - final TableName tableName = TableName.valueOf("testGetTableRegions"); - - int expectedRegions = 10; - - // Use 80 bit numbers to make sure we aren't limited - byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; - - - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin.createTable(desc, startKey, endKey, expectedRegions); - - List RegionInfos = admin.getTableRegions(tableName); - - assertEquals("Tried to create " + expectedRegions + " regions " + - "but only found " + RegionInfos.size(), - expectedRegions, RegionInfos.size()); - } - - @Test (timeout=300000) - public void testMoveToPreviouslyAssignedRS() throws IOException, InterruptedException { - byte[] tableName = Bytes.toBytes("testMoveToPreviouslyAssignedRS"); - MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - HMaster master = cluster.getMaster(); - HBaseAdmin localAdmin = createTable(tableName); - List tableRegions = localAdmin.getTableRegions(tableName); - HRegionInfo hri = tableRegions.get(0); - AssignmentManager am = master.getAssignmentManager(); - assertTrue("Region " + hri.getRegionNameAsString() - + " should be assigned properly", am.waitForAssignment(hri)); - ServerName server = am.getRegionStates().getRegionServerOfRegion(hri); - localAdmin.move(hri.getEncodedNameAsBytes(), Bytes.toBytes(server.getServerName())); - assertEquals("Current region server and region server before move should be same.", server, - am.getRegionStates().getRegionServerOfRegion(hri)); - } - - @Test (timeout=300000) - public void testHLogRollWriting() throws Exception { - setUpforLogRolling(); - String className = this.getClass().getName(); - StringBuilder v = new StringBuilder(className); - while (v.length() < 1000) { - v.append(className); - } - byte[] value = Bytes.toBytes(v.toString()); - HRegionServer regionServer = startAndWriteData(TableName.valueOf("TestLogRolling"), value); - LOG.info("after writing there are " - + HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()) + " log files"); - - // flush all regions - - List regions = new ArrayList(regionServer - .getOnlineRegionsLocalContext()); - for (HRegion r : regions) { - r.flushcache(); - } - admin.rollHLogWriter(regionServer.getServerName().getServerName()); - int count = HLogUtilsForTests.getNumRolledLogFiles(regionServer.getWAL()); - LOG.info("after flushing all regions and rolling logs there are " + - count + " log files"); - assertTrue(("actual count: " + count), count <= 2); - } - - private void setUpforLogRolling() { - // Force a region split after every 768KB - TEST_UTIL.getConfiguration().setLong(HConstants.HREGION_MAX_FILESIZE, - 768L * 1024L); - - // We roll the log after every 32 writes - TEST_UTIL.getConfiguration().setInt("hbase.regionserver.maxlogentries", 32); - - TEST_UTIL.getConfiguration().setInt( - "hbase.regionserver.logroll.errors.tolerated", 2); - TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 10 * 1000); - - // For less frequently updated regions flush after every 2 flushes - TEST_UTIL.getConfiguration().setInt( - "hbase.hregion.memstore.optionalflushcount", 2); - - // We flush the cache after every 8192 bytes - TEST_UTIL.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, - 8192); - - // Increase the amount of time between client retries - TEST_UTIL.getConfiguration().setLong("hbase.client.pause", 10 * 1000); - - // Reduce thread wake frequency so that other threads can get - // a chance to run. - TEST_UTIL.getConfiguration().setInt(HConstants.THREAD_WAKE_FREQUENCY, - 2 * 1000); - - /**** configuration for testLogRollOnDatanodeDeath ****/ - // make sure log.hflush() calls syncFs() to open a pipeline - TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); - // lower the namenode & datanode heartbeat so the namenode - // quickly detects datanode failures - TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000); - TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1); - // the namenode might still try to choose the recently-dead datanode - // for a pipeline, so try to a new pipeline multiple times - TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 30); - TEST_UTIL.getConfiguration().setInt( - "hbase.regionserver.hlog.tolerable.lowreplication", 2); - TEST_UTIL.getConfiguration().setInt( - "hbase.regionserver.hlog.lowreplication.rolllimit", 3); - } - - private HRegionServer startAndWriteData(TableName tableName, byte[] value) - throws IOException, InterruptedException { - // When the hbase:meta table can be opened, the region servers are running - new HTable( - TEST_UTIL.getConfiguration(), TableName.META_TABLE_NAME).close(); - - // Create the test table and open it - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - admin.createTable(desc); - Table table = new HTable(TEST_UTIL.getConfiguration(), tableName); - - HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(tableName); - for (int i = 1; i <= 256; i++) { // 256 writes should cause 8 log rolls - Put put = new Put(Bytes.toBytes("row" + String.format("%1$04d", i))); - put.add(HConstants.CATALOG_FAMILY, null, value); - table.put(put); - if (i % 32 == 0) { - // After every 32 writes sleep to let the log roller run - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - // continue - } - } - } - - table.close(); - return regionServer; - } - - /** - * HBASE-4417 checkHBaseAvailable() doesn't close zk connections - */ - @Test (timeout=300000) - public void testCheckHBaseAvailableClosesConnection() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - - int initialCount = HConnectionTestingUtility.getConnectionCount(); - HBaseAdmin.checkHBaseAvailable(conf); - int finalCount = HConnectionTestingUtility.getConnectionCount(); - - Assert.assertEquals(initialCount, finalCount) ; - } - - /** - * Check that we have an exception if the cluster is not there. - */ - @Test (timeout=300000) - public void testCheckHBaseAvailableWithoutCluster() { - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - - // Change the ZK address to go to something not used. - conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, - conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, 9999)+10); - - int initialCount = HConnectionTestingUtility.getConnectionCount(); - - long start = System.currentTimeMillis(); - try { - HBaseAdmin.checkHBaseAvailable(conf); - assertTrue(false); - } catch (MasterNotRunningException ignored) { - } catch (ZooKeeperConnectionException ignored) { - } catch (ServiceException ignored) { - } catch (IOException ignored) { - } - long end = System.currentTimeMillis(); - - int finalCount = HConnectionTestingUtility.getConnectionCount(); - - Assert.assertEquals(initialCount, finalCount) ; - - LOG.info("It took "+(end-start)+" ms to find out that" + - " HBase was not available"); - } - - @Test (timeout=300000) - public void testDisableCatalogTable() throws Exception { - try { - this.admin.disableTable(TableName.META_TABLE_NAME); - fail("Expected to throw ConstraintException"); - } catch (ConstraintException e) { - } - // Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table - // actually getting disabled by the disableTable() call. - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testDisableCatalogTable".getBytes())); - HColumnDescriptor hcd = new HColumnDescriptor("cf1".getBytes()); - htd.addFamily(hcd); - TEST_UTIL.getHBaseAdmin().createTable(htd); - } - - @Test (timeout=300000) - public void testIsEnabledOrDisabledOnUnknownTable() throws Exception { - try { - admin.isTableEnabled(TableName.valueOf("unkownTable")); - fail("Test should fail if isTableEnabled called on unknown table."); - } catch (IOException e) { - } - - try { - admin.isTableDisabled(TableName.valueOf("unkownTable")); - fail("Test should fail if isTableDisabled called on unknown table."); - } catch (IOException e) { - } - } - - @Test (timeout=300000) - public void testGetRegion() throws Exception { - // We use actual HBaseAdmin instance instead of going via Admin interface in - // here because makes use of an internal HBA method (TODO: Fix.). - HBaseAdmin rawAdmin = new HBaseAdmin(TEST_UTIL.getConfiguration()); - - final TableName tableName = TableName.valueOf("testGetRegion"); - LOG.info("Started " + tableName); - HTable t = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); - TEST_UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); - - HRegionLocation regionLocation = t.getRegionLocation("mmm"); - HRegionInfo region = regionLocation.getRegionInfo(); - byte[] regionName = region.getRegionName(); - Pair pair = rawAdmin.getRegion(regionName); - assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); - pair = rawAdmin.getRegion(region.getEncodedNameAsBytes()); - assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java deleted file mode 100644 index 709e94ba2a48..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java +++ /dev/null @@ -1,313 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.client; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.exceptions.PreemptiveFastFailException; -import org.apache.hadoop.hbase.testclassification.ClientTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({MediumTests.class, ClientTests.class}) -public class TestFastFail { - final Log LOG = LogFactory.getLog(getClass()); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static byte[] FAMILY = Bytes.toBytes("testFamily"); - private static final Random random = new Random(); - private static int SLAVES = 3; - private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); - private static final int SLEEPTIME = 1000; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - TEST_UTIL.startMiniCluster(SLAVES); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - // Nothing to do. - } - - /** - * @throws java.lang.Exception - */ - @After - public void tearDown() throws Exception { - // Nothing to do. - } - - @Test - public void testFastFail() throws IOException, InterruptedException { - Admin admin = TEST_UTIL.getHBaseAdmin(); - - final String tableName = "testClientRelearningExperiment"; - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes - .toBytes(tableName))); - desc.addFamily(new HColumnDescriptor(FAMILY)); - admin.createTable(desc, Bytes.toBytes("aaaa"), Bytes.toBytes("zzzz"), 32); - final long numRows = 1000; - - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setLong(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, SLEEPTIME * 100); - conf.setInt(HConstants.HBASE_CLIENT_PAUSE, SLEEPTIME / 10); - conf.setBoolean(HConstants.HBASE_CLIENT_FAST_FAIL_MODE_ENABLED, true); - conf.setLong(HConstants.HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS, 0); - conf.setClass(HConstants.HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL, - MyPreemptiveFastFailInterceptor.class, - PreemptiveFastFailInterceptor.class); - - final Connection connection = ConnectionFactory.createConnection(conf); - - /** - * Write numRows worth of data, so that the workers can arbitrarily read. - */ - try (Table table = connection.getTable(TableName.valueOf(tableName));) { - writeData(table, numRows); - } - - /** - * The number of threads that are going to perform actions against the test - * table. - */ - int nThreads = 200; - ExecutorService service = Executors.newFixedThreadPool(nThreads); - final CountDownLatch continueOtherHalf = new CountDownLatch(1); - final CountDownLatch doneHalfway = new CountDownLatch(nThreads); - - final AtomicInteger numSuccessfullThreads = new AtomicInteger(0); - final AtomicInteger numFailedThreads = new AtomicInteger(0); - - // The total time taken for the threads to perform the second put; - final AtomicLong totalTimeTaken = new AtomicLong(0); - final AtomicInteger numBlockedWorkers = new AtomicInteger(0); - final AtomicInteger numPreemptiveFastFailExceptions = new AtomicInteger(0); - - List> futures = new ArrayList>(); - for (int i = 0; i < nThreads; i++) { - futures.add(service.submit(new Callable() { - /** - * The workers are going to perform a couple of reads. The second read - * will follow the killing of a regionserver so that we make sure that - * some of threads go into PreemptiveFastFailExcception - */ - public Boolean call() throws Exception { - try (Table table = connection.getTable(TableName.valueOf(tableName))) { - Thread.sleep(Math.abs(random.nextInt()) % 100); // Add some jitter - // here - byte[] row = longToByteArrayKey(Math.abs(random.nextLong()) - % numRows); - Get g = new Get(row); - g.addColumn(FAMILY, QUALIFIER); - try { - table.get(g); - } catch (Exception e) { - LOG.debug("Get failed : ", e); - doneHalfway.countDown(); - return false; - } - - // Done with one get, proceeding to do the next one. - doneHalfway.countDown(); - continueOtherHalf.await(); - - long startTime = System.currentTimeMillis(); - g = new Get(row); - g.addColumn(FAMILY, QUALIFIER); - try { - table.get(g); - // The get was successful - numSuccessfullThreads.addAndGet(1); - } catch (Exception e) { - if (e instanceof PreemptiveFastFailException) { - // We were issued a PreemptiveFastFailException - numPreemptiveFastFailExceptions.addAndGet(1); - } - // Irrespective of PFFE, the request failed. - numFailedThreads.addAndGet(1); - return false; - } finally { - long enTime = System.currentTimeMillis(); - totalTimeTaken.addAndGet(enTime - startTime); - if ((enTime - startTime) >= SLEEPTIME) { - // Considering the slow workers as the blockedWorkers. - // This assumes that the threads go full throttle at performing - // actions. In case the thread scheduling itself is as slow as - // SLEEPTIME, then this test might fail and so, we might have - // set it to a higher number on slower machines. - numBlockedWorkers.addAndGet(1); - } - } - return true; - } catch (Exception e) { - LOG.error("Caught unknown exception", e); - doneHalfway.countDown(); - return false; - } - } - })); - } - - doneHalfway.await(); - - ClusterStatus status = TEST_UTIL.getHBaseCluster().getClusterStatus(); - - // Kill a regionserver - TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer().stop(); - TEST_UTIL.getHBaseCluster().getRegionServer(0).stop("Testing"); - - // Let the threads continue going - continueOtherHalf.countDown(); - - Thread.sleep(2 * SLEEPTIME); - // Restore the cluster - TEST_UTIL.getHBaseCluster().restoreClusterStatus(status); - - int numThreadsReturnedFalse = 0; - int numThreadsReturnedTrue = 0; - int numThreadsThrewExceptions = 0; - for (Future f : futures) { - try { - numThreadsReturnedTrue += f.get() ? 1 : 0; - numThreadsReturnedFalse += f.get() ? 0 : 1; - } catch (Exception e) { - numThreadsThrewExceptions++; - } - } - LOG.debug("numThreadsReturnedFalse:" - + numThreadsReturnedFalse - + " numThreadsReturnedTrue:" - + numThreadsReturnedTrue - + " numThreadsThrewExceptions:" - + numThreadsThrewExceptions - + " numFailedThreads:" - + numFailedThreads.get() - + " numSuccessfullThreads:" - + numSuccessfullThreads.get() - + " numBlockedWorkers:" - + numBlockedWorkers.get() - + " totalTimeWaited: " - + totalTimeTaken.get() - / (numBlockedWorkers.get() == 0 ? Long.MAX_VALUE : numBlockedWorkers - .get()) + " numPFFEs: " + numPreemptiveFastFailExceptions.get()); - - assertEquals("The expected number of all the successfull and the failed " - + "threads should equal the total number of threads that we spawned", - nThreads, numFailedThreads.get() + numSuccessfullThreads.get()); - assertEquals( - "All the failures should be coming from the secondput failure", - numFailedThreads.get(), numThreadsReturnedFalse); - assertEquals("Number of threads that threw execution exceptions " - + "otherwise should be 0", numThreadsThrewExceptions, 0); - assertEquals("The regionservers that returned true should equal to the" - + " number of successful threads", numThreadsReturnedTrue, - numSuccessfullThreads.get()); - assertTrue( - "There should be atleast one thread that retried instead of failing", - MyPreemptiveFastFailInterceptor.numBraveSouls.get() > 0); - assertTrue( - "There should be atleast one PreemptiveFastFail exception," - + " otherwise, the test makes little sense." - + "numPreemptiveFastFailExceptions: " - + numPreemptiveFastFailExceptions.get(), - numPreemptiveFastFailExceptions.get() > 0); - assertTrue( - "Only few thread should ideally be waiting for the dead " - + "regionserver to be coming back. numBlockedWorkers:" - + numBlockedWorkers.get() + " threads that retried : " - + MyPreemptiveFastFailInterceptor.numBraveSouls.get(), - numBlockedWorkers.get() <= MyPreemptiveFastFailInterceptor.numBraveSouls - .get()); - } - - public static class MyPreemptiveFastFailInterceptor extends - PreemptiveFastFailInterceptor { - public static AtomicInteger numBraveSouls = new AtomicInteger(); - - @Override - protected boolean shouldRetryInspiteOfFastFail(FailureInfo fInfo) { - boolean ret = super.shouldRetryInspiteOfFastFail(fInfo); - if (ret) - numBraveSouls.addAndGet(1); - return ret; - } - - public MyPreemptiveFastFailInterceptor(Configuration conf) { - super(conf); - } - } - - private byte[] longToByteArrayKey(long rowKey) { - return LoadTestKVGenerator.md5PrefixedKey(rowKey).getBytes(); - } - - public void writeData(Table table, long numRows) throws IOException, - InterruptedException { - table.flushCommits(); - for (long i = 0; i < numRows; i++) { - byte[] rowKey = longToByteArrayKey(i); - Put put = new Put(rowKey); - byte[] value = rowKey; // value is the same as the row key - put.add(FAMILY, QUALIFIER, value); - table.put(put); - } - LOG.info("Written all puts."); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 86ba1895dc6d..bfece260db16 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -75,7 +75,6 @@ import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.hadoop.hbase.filter.LongComparator; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; @@ -133,11 +132,9 @@ public class TestFromClientSide { */ @BeforeClass public static void setUpBeforeClass() throws Exception { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); Configuration conf = TEST_UTIL.getConfiguration(); conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MultiRowMutationEndpoint.class.getName()); @@ -750,37 +747,6 @@ public void testFilters() throws Exception { scanner.close(); } - @Test - public void testFilterWithLongCompartor() throws Exception { - byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor"); - Table ht = TEST_UTIL.createTable(TABLE, FAMILY); - byte [][] ROWS = makeN(ROW, 10); - byte [][] values = new byte[10][]; - for (int i = 0; i < 10; i ++) { - values[i] = Bytes.toBytes(100L * i); - } - for(int i = 0; i < 10; i ++) { - Put put = new Put(ROWS[i]); - put.setDurability(Durability.SKIP_WAL); - put.add(FAMILY, QUALIFIER, values[i]); - ht.put(put); - } - Scan scan = new Scan(); - scan.addFamily(FAMILY); - Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER, - new LongComparator(500)); - scan.setFilter(filter); - ResultScanner scanner = ht.getScanner(scan); - int expectedIndex = 0; - for(Result result : ht.getScanner(scan)) { - assertEquals(result.size(), 1); - assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500); - expectedIndex++; - } - assertEquals(expectedIndex, 4); - scanner.close(); -} - @Test public void testKeyOnlyFilter() throws Exception { byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index fd32eefb5dd8..31bc13595831 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -222,7 +222,7 @@ public void testAdvancedConfigOverride() throws Exception { LOG.info("hbase.hstore.compaction.min should now be 2"); HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2)); - htd.modifyFamily(hcd); + htd.addFamily(hcd); admin.modifyTable(TABLE, htd); while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) { LOG.debug(st.getFirst() + " regions left to update"); @@ -257,7 +257,7 @@ public void testAdvancedConfigOverride() throws Exception { LOG.info("hbase.hstore.compaction.min should now be 5"); hcd = new HColumnDescriptor(htd.getFamily(FAMILY)); hcd.setValue("hbase.hstore.compaction.min", null); - htd.modifyFamily(hcd); + htd.addFamily(hcd); admin.modifyTable(TABLE, htd); while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) { LOG.debug(st.getFirst() + " regions left to update"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java index 20828ecdd23e..b911f6e06031 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java @@ -60,7 +60,10 @@ @Category({MediumTests.class, FlakeyTests.class}) public class TestMultiParallel { private static final Log LOG = LogFactory.getLog(TestMultiParallel.class); - + { + ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); + } private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final byte[] VALUE = Bytes.toBytes("value"); private static final byte[] QUALIFIER = Bytes.toBytes("qual"); @@ -73,11 +76,9 @@ public class TestMultiParallel { private static final int slaves = 5; // also used for testing HTable pool size @BeforeClass public static void beforeClass() throws Exception { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); UTIL.startMiniCluster(slaves); HTable t = UTIL.createTable(TEST_TABLE, Bytes.toBytes(FAMILY)); UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java index 3f983ed161df..599accf10605 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALObserver.java @@ -112,9 +112,7 @@ public boolean preWALWrite(ObserverContext env, cell.getValueArray()[cell.getValueOffset()] += 1; } } - if (null != row) { - cells.add(new KeyValue(row, addedFamily, addedQualifier)); - } + cells.add(new KeyValue(row, addedFamily, addedQualifier)); if (deletedCell != null) { LOG.debug("About to delete a KeyValue from WALEdit."); cells.remove(deletedCell); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index 140c3b9d6c24..2fe121df64d3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -531,19 +531,6 @@ public void testMasterCoprocessorsReported() { assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors); } - @Test - public void testFindCoprocessors() { - // HBASE 12277: - CoprocessorHost masterCpHost = - TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost(); - - List masterObservers = masterCpHost.findCoprocessors(MasterObserver.class); - - assertTrue(masterObservers != null && masterObservers.size() > 0); - assertEquals(masterCoprocessor.getSimpleName(), - masterObservers.get(0).getClass().getSimpleName()); - } - private void waitForTable(TableName name) throws InterruptedException, IOException { // First wait until all regions are online TEST_UTIL.waitTableEnabled(name); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java index 061068cbaa9c..81db6b40e962 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java @@ -198,7 +198,7 @@ public boolean isAborted() { // Test (part of the) output that should have be printed by master when it aborts: // (namely the part that shows the set of loaded coprocessors). // In this test, there is only a single coprocessor (BuggyMasterObserver). - assertTrue(HMaster.getLoadedCoprocessors(). + assertTrue(master.getLoadedCoprocessors(). contains(TestMasterCoprocessorExceptionWithAbort.BuggyMasterObserver.class.getName())); CreateTableThread createTableThread = new CreateTableThread(UTIL); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 2bb56b5136e6..75fe6d2e410a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -229,36 +229,6 @@ public void testWALObserverWriteToWAL() throws Exception { assertTrue(cp.isPostWALWriteCalled()); } - /** - * Coprocessors shouldn't get notice of empty waledits. - */ - @Test - public void testEmptyWALEditAreNotSeen() throws Exception { - final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); - final AtomicLong sequenceId = new AtomicLong(0); - - HLog log = HLogFactory.createHLog(this.fs, hbaseRootDir, - TestWALObserver.class.getName(), this.conf); - try { - SampleRegionWALObserver cp = getCoprocessor(log); - - cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null); - - assertFalse(cp.isPreWALWriteCalled()); - assertFalse(cp.isPostWALWriteCalled()); - - final long now = EnvironmentEdgeManager.currentTime(); - log.append(hri, hri.getTable(), new WALEdit(), now, htd, sequenceId); - log.sync(); - - assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled()); - assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled()); - } finally { - log.closeAndDelete(); - } - } - /** * Test WAL replay behavior with WALObserver. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java index c64890730648..319b5f5a1fd8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java @@ -141,6 +141,7 @@ public void setUp() throws Exception { htd.addFamily(new HColumnDescriptor(FAMILIES_1[1])); htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[0])); htd.addFamily(new HColumnDescriptor(NEW_FAMILIES[1])); + htd.addFamily(new HColumnDescriptor(FAMILIES_1[1])); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java index b4e364defe6d..17f8c83e3cb8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.regex.Pattern; import org.apache.hadoop.hbase.KeyValue; @@ -96,25 +95,6 @@ private Filter regexFilterNew(Pattern pattern) { new RegexStringComparator(pattern.pattern(), pattern.flags())); } - @Test - public void testLongComparator() throws IOException { - Filter filter = new SingleColumnValueFilter(COLUMN_FAMILY, - COLUMN_QUALIFIER, CompareOp.GREATER, new LongComparator(100L)); - KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, - Bytes.toBytes(1L)); - assertTrue("less than", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); - filter.reset(); - - kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, - Bytes.toBytes(100L)); - assertTrue("Equals 100", filter.filterKeyValue(kv) == Filter.ReturnCode.NEXT_ROW); - filter.reset(); - - kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, - Bytes.toBytes(120L)); - assertTrue("include 120", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); - } - private void basicFilterTests(SingleColumnValueFilter filter) throws Exception { KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index de842926670d..9672764b5e07 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -39,7 +39,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.LruCachedBlock; @@ -373,7 +372,7 @@ public void testMutations(){ byte[] row = new byte[] { 0 }; cl = Put.class; - actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY); + actual = new Put(row).MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY); expected = ClassSize.estimateBase(cl, false); //The actual TreeMap is not included in the above calculation expected += ClassSize.align(ClassSize.TREEMAP); @@ -383,7 +382,7 @@ public void testMutations(){ } cl = Delete.class; - actual = Mutation.MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY); + actual = new Delete(row).MUTATION_OVERHEAD + ClassSize.align(ClassSize.ARRAY); expected = ClassSize.estimateBase(cl, false); //The actual TreeMap is not included in the above calculation expected += ClassSize.align(ClassSize.TREEMAP); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java index 80266af785a8..00c6aa570fca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java @@ -49,6 +49,10 @@ @Category({IOTests.class, SmallTests.class}) public class TestChecksum { + // change this value to activate more logs + private static final boolean detailedLogging = true; + private static final boolean[] BOOLEAN_VALUES = new boolean[] { false, true }; + private static final Log LOG = LogFactory.getLog(TestHFileBlock.class); static final Compression.Algorithm[] COMPRESSION_ALGORITHMS = { @@ -114,7 +118,7 @@ protected void testChecksumCorruptionInternals(boolean useTags) throws IOExcepti .withIncludesTags(useTags) .withHBaseCheckSum(true) .build(); - HFileBlock.FSReader hbr = new FSReaderImplTest(is, totalSize, fs, path, meta); + HFileBlock.FSReader hbr = new FSReaderV2Test(is, totalSize, fs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, -1, pread); b.sanityCheck(); assertEquals(4936, b.getUncompressedSizeWithoutHeader()); @@ -156,7 +160,7 @@ protected void testChecksumCorruptionInternals(boolean useTags) throws IOExcepti HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false); assertEquals(false, newfs.useHBaseChecksum()); is = new FSDataInputStreamWrapper(newfs, path); - hbr = new FSReaderImplTest(is, totalSize, newfs, path, meta); + hbr = new FSReaderV2Test(is, totalSize, newfs, path, meta); b = hbr.readBlockData(0, -1, -1, pread); is.close(); b.sanityCheck(); @@ -239,7 +243,7 @@ protected void testChecksumInternals(boolean useTags) throws IOException { .withHBaseCheckSum(true) .withBytesPerCheckSum(bytesPerChecksum) .build(); - HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper( + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper( is, nochecksum), totalSize, hfs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, -1, pread); is.close(); @@ -279,8 +283,8 @@ private void validateData(DataInputStream in) throws IOException { * reading data from hfiles. This should trigger the hdfs level * checksum validations. */ - static private class FSReaderImplTest extends HFileBlock.FSReaderImpl { - public FSReaderImplTest(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs, + static private class FSReaderV2Test extends HFileBlock.FSReaderV2 { + public FSReaderV2Test(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs, Path path, HFileContext meta) throws IOException { super(istream, fileSize, (HFileSystem) fs, path, meta); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 3855629781fb..4523b2aee365 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -45,9 +45,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -70,12 +67,12 @@ public class TestHFile extends HBaseTestCase { private static CacheConfig cacheConf = null; private Map startingMetrics; - @Before + @Override public void setUp() throws Exception { super.setUp(); } - @After + @Override public void tearDown() throws Exception { super.tearDown(); } @@ -86,7 +83,6 @@ public void tearDown() throws Exception { * Test all features work reasonably when hfile is empty of entries. * @throws IOException */ - @Test public void testEmptyHFile() throws IOException { if (cacheConf == null) cacheConf = new CacheConfig(conf); Path f = new Path(ROOT_DIR, getName()); @@ -103,7 +99,6 @@ public void testEmptyHFile() throws IOException { /** * Create 0-length hfile and show that it fails */ - @Test public void testCorrupt0LengthHFile() throws IOException { if (cacheConf == null) cacheConf = new CacheConfig(conf); Path f = new Path(ROOT_DIR, getName()); @@ -137,7 +132,6 @@ public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOExce /** * Create a truncated hfile and verify that exception thrown. */ - @Test public void testCorruptTruncatedHFile() throws IOException { if (cacheConf == null) cacheConf = new CacheConfig(conf); Path f = new Path(ROOT_DIR, getName()); @@ -287,13 +281,11 @@ void basicWithSomeCodec(String codec, boolean useTags) throws IOException { fs.delete(ncTFile, true); } - @Test public void testTFileFeatures() throws IOException { testTFilefeaturesInternals(false); testTFilefeaturesInternals(true); } - @Test protected void testTFilefeaturesInternals(boolean useTags) throws IOException { basicWithSomeCodec("none", useTags); basicWithSomeCodec("gz", useTags); @@ -361,13 +353,11 @@ private void metablocks(final String compress) throws Exception { } // test meta blocks for tfiles - @Test public void testMetaBlocks() throws Exception { metablocks("none"); metablocks("gz"); } - @Test public void testNullMetaBlocks() throws Exception { if (cacheConf == null) cacheConf = new CacheConfig(conf); for (Compression.Algorithm compressAlgo : diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java index eb1f1bb59fad..254af72b241d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java @@ -313,7 +313,7 @@ protected void testReaderV2Internals() throws IOException { .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTag) .withCompression(algo).build(); - HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, totalSize, meta); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, totalSize, meta); HFileBlock b = hbr.readBlockData(0, -1, -1, pread); is.close(); assertEquals(0, HFile.getChecksumFailuresCount()); @@ -326,7 +326,7 @@ protected void testReaderV2Internals() throws IOException { if (algo == GZ) { is = fs.open(path); - hbr = new HFileBlock.FSReaderImpl(is, totalSize, meta); + hbr = new HFileBlock.FSReaderV2(is, totalSize, meta); b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE + b.totalChecksumBytes(), -1, pread); assertEquals(expected, b); @@ -412,7 +412,7 @@ private void testInternals() throws IOException { .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTag) .build(); - HFileBlock.FSReaderImpl hbr = new HFileBlock.FSReaderImpl(is, totalSize, meta); + HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, totalSize, meta); hbr.setDataBlockEncoder(dataBlockEncoder); hbr.setIncludesMemstoreTS(includesMemstoreTS); HFileBlock blockFromHFile, blockUnpacked; @@ -540,7 +540,7 @@ protected void testPreviousOffsetInternals() throws IOException { .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTag) .withCompression(algo).build(); - HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, totalSize, meta); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, totalSize, meta); long curOffset = 0; for (int i = 0; i < NUM_TEST_BLOCKS; ++i) { if (!pread) { @@ -725,7 +725,7 @@ protected void testConcurrentReadingInternals() throws IOException, .withIncludesTags(includesTag) .withCompression(compressAlgo) .build(); - HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, fileSize, meta); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(is, fileSize, meta); Executor exec = Executors.newFixedThreadPool(NUM_READER_THREADS); ExecutorCompletionService ecs = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java index fc44f3cf0cde..da6761e44bb3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java @@ -197,8 +197,8 @@ public void testReaderV2() throws IOException { .withIncludesTags(includesTag) .withCompression(algo) .build(); - HFileBlock.FSReader hbr = - new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(is), totalSize, fs, path, meta); + HFileBlock.FSReader hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is), + totalSize, fs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, -1, pread); is.close(); @@ -210,7 +210,7 @@ public void testReaderV2() throws IOException { if (algo == GZ) { is = fs.open(path); - hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(is), totalSize, fs, path, + hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is), totalSize, fs, path, meta); b = hbr.readBlockData(0, 2173 + HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM + b.totalChecksumBytes(), -1, pread); @@ -292,7 +292,7 @@ public void testDataBlockEncoding() throws IOException { .withIncludesTags(includesTag) .withCompression(algo) .build(); - HFileBlock.FSReaderImpl hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(is), + HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(new FSDataInputStreamWrapper(is), totalSize, fs, path, meta); hbr.setDataBlockEncoder(dataBlockEncoder); hbr.setIncludesMemstoreTS(includesMemstoreTS); @@ -740,7 +740,7 @@ public HFileBlock getBlockForCaching() { .build(); return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), getUncompressedSizeWithoutHeader(), prevOffset, - getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset, + getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset, getOnDiskSizeWithoutHeader(), meta); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index c0f2fed94a36..831c6d3c3316 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -197,7 +197,7 @@ public void readIndex(boolean useTags) throws IOException { .withIncludesTags(useTags) .withCompression(compr) .build(); - HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(istream, fs.getFileStatus(path) + HFileBlock.FSReader blockReader = new HFileBlock.FSReaderV2(istream, fs.getFileStatus(path) .getLen(), meta); BlockReaderWrapper brw = new BlockReaderWrapper(blockReader); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java index 0cb3c3cfdf28..b1a4d570521f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java @@ -93,7 +93,7 @@ private int writeBlock(FSDataOutputStream os, HFileContext fileContext, int size return hbw.getOnDiskSizeWithHeader(); } - private long readAndVerifyBlock(long pos, HFileContext ctx, HFileBlock.FSReaderImpl hbr, int size) + private long readAndVerifyBlock(long pos, HFileContext ctx, HFileBlock.FSReaderV2 hbr, int size) throws IOException { HFileBlock b = hbr.readBlockData(pos, -1, -1, false); assertEquals(0, HFile.getChecksumFailuresCount()); @@ -139,7 +139,7 @@ public void testDataBlockEncryption() throws IOException { } FSDataInputStream is = fs.open(path); try { - HFileBlock.FSReaderImpl hbr = new HFileBlock.FSReaderImpl(is, totalSize, fileContext); + HFileBlock.FSReaderV2 hbr = new HFileBlock.FSReaderV2(is, totalSize, fileContext); long pos = 0; for (int i = 0; i < blocks; i++) { pos += readAndVerifyBlock(pos, fileContext, hbr, blockSizes[i]); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java index 42e918a15239..12b0639c70a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java @@ -146,7 +146,7 @@ private void writeDataAndReadFromHFile(Path hfilePath, .withCompression(compressAlgo) .build(); - HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, meta); + HFileBlock.FSReader blockReader = new HFileBlock.FSReaderV2(fsdis, fileSize, meta); // Comparator class name is stored in the trailer in version 2. KVComparator comparator = trailer.createComparator(); HFileBlockIndex.BlockIndexReader dataBlockIndexReader = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java index f96e8ef757d1..471a44de955b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java @@ -176,7 +176,7 @@ private void writeDataAndReadFromHFile(Path hfilePath, .withIncludesTags(useTags) .withHBaseCheckSum(true).build(); HFileBlock.FSReader blockReader = - new HFileBlock.FSReaderImpl(fsdis, fileSize, meta); + new HFileBlock.FSReaderV2(fsdis, fileSize, meta); // Comparator class name is stored in the trailer in version 2. KVComparator comparator = trailer.createComparator(); HFileBlockIndex.BlockIndexReader dataBlockIndexReader = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java index c1a506199e96..e31ebb90daa1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java @@ -60,7 +60,8 @@ public class TestScannerSelectionUsingTTL { private static final Log LOG = LogFactory.getLog(TestScannerSelectionUsingTTL.class); - private static final HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU(); + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility().createLocalHTU(); private static TableName TABLE = TableName.valueOf("myTable"); private static String FAMILY = "myCF"; private static byte[] FAMILY_BYTES = Bytes.toBytes(FAMILY); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index b9a126f84410..b7be1bb18785 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -72,7 +71,7 @@ static String toRowStr(Cell kv) { } Path makeNewFile(TagUsage tagUsage) throws IOException { - Path ncTFile = new Path(testDir, "basic.hfile"); + Path ncTFile = new Path(this.testDir, "basic.hfile"); if (tagUsage != TagUsage.NO_TAG) { conf.setInt("hfile.format.version", 3); } else { @@ -98,7 +97,6 @@ Path makeNewFile(TagUsage tagUsage) throws IOException { return ncTFile; } - @Test public void testSeekBefore() throws Exception { testSeekBeforeInternals(TagUsage.NO_TAG); testSeekBeforeInternals(TagUsage.ONLY_TAG); @@ -140,7 +138,6 @@ protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException { reader.close(); } - @Test public void testSeekBeforeWithReSeekTo() throws Exception { testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG); testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG); @@ -230,7 +227,6 @@ protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOE assertEquals("k", toRowStr(scanner.getKeyValue())); } - @Test public void testSeekTo() throws Exception { testSeekToInternals(TagUsage.NO_TAG); testSeekToInternals(TagUsage.ONLY_TAG); @@ -250,8 +246,8 @@ protected void testSeekToInternals(TagUsage tagUsage) throws IOException { assertEquals("c", toRowStr(scanner.getKeyValue())); // Across a block boundary now. - // 'h' does not exist so we will get a '1' back for not found. - assertEquals(0, scanner.seekTo(toKV("i", tagUsage))); + // h goes to the next block + assertEquals(-2, scanner.seekTo(toKV("h", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); assertEquals(1, scanner.seekTo(toKV("l", tagUsage))); @@ -259,8 +255,6 @@ protected void testSeekToInternals(TagUsage tagUsage) throws IOException { reader.close(); } - - @Test public void testBlockContainingKey() throws Exception { testBlockContainingKeyInternals(TagUsage.NO_TAG); testBlockContainingKeyInternals(TagUsage.ONLY_TAG); @@ -274,6 +268,7 @@ protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOExcep HFileBlockIndex.BlockIndexReader blockIndexReader = reader.getDataBlockIndexReader(); System.out.println(blockIndexReader.toString()); + int klen = toKV("a", tagUsage).getKey().length; // falls before the start of the file. assertEquals(-1, blockIndexReader.rootBlockContainingKey( toKV("a", tagUsage))); @@ -285,7 +280,8 @@ protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOExcep toKV("e", tagUsage))); assertEquals(0, blockIndexReader.rootBlockContainingKey( toKV("g", tagUsage))); - assertEquals(1, blockIndexReader.rootBlockContainingKey(toKV("h", tagUsage))); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("h", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( toKV("i", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( @@ -296,4 +292,7 @@ protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOExcep toKV("l", tagUsage))); reader.close(); } -} \ No newline at end of file + + +} + diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java index 443ec781ba37..af764f45e11d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcMetrics.java @@ -48,8 +48,8 @@ public void testFactory() { assertEquals("Master,sub=IPC", masterSource.getMetricsJmxContext()); assertEquals("RegionServer,sub=IPC", rsSource.getMetricsJmxContext()); - assertEquals("Master", masterSource.getMetricsName()); - assertEquals("RegionServer", rsSource.getMetricsName()); + assertEquals("IPC", masterSource.getMetricsName()); + assertEquals("IPC", rsSource.getMetricsName()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestDriver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestDriver.java index ab6a86d2616c..a268b766d7f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestDriver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestDriver.java @@ -32,6 +32,7 @@ public class TestDriver { @Test + @SuppressWarnings("deprecation") public void testDriverMainMethod() throws Throwable { ProgramDriver programDriverMock = mock(ProgramDriver.class); Driver.setProgramDriver(programDriverMock); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java index 1dab9ddc68f2..092a18fefee8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java @@ -23,7 +23,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; @@ -33,7 +32,6 @@ import org.apache.hadoop.hbase.util.LauncherSecurityManager; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.GenericOptionsParser; -import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -158,56 +156,4 @@ public void testCellCounterMain() throws Exception { System.setSecurityManager(SECURITY_MANAGER); } } - - /** - * Test CellCounter for complete table all data should print to output - */ - @Test(timeout = 300000) - public void testCellCounterForCompleteTable() throws Exception { - String sourceTable = "testCellCounterForCompleteTable"; - String outputPath = OUTPUT_DIR + sourceTable; - LocalFileSystem localFileSystem = new LocalFileSystem(); - Path outputDir = - new Path(outputPath).makeQualified(localFileSystem.getUri(), - localFileSystem.getWorkingDirectory()); - byte[][] families = { FAMILY_A, FAMILY_B }; - Table t = UTIL.createTable(Bytes.toBytes(sourceTable), families); - try { - Put p = new Put(ROW1); - p.add(FAMILY_A, QUALIFIER, now, Bytes.toBytes("Data11")); - p.add(FAMILY_B, QUALIFIER, now + 1, Bytes.toBytes("Data12")); - p.add(FAMILY_A, QUALIFIER, now + 2, Bytes.toBytes("Data13")); - t.put(p); - p = new Put(ROW2); - p.add(FAMILY_B, QUALIFIER, now, Bytes.toBytes("Dat21")); - p.add(FAMILY_A, QUALIFIER, now + 1, Bytes.toBytes("Data22")); - p.add(FAMILY_B, QUALIFIER, now + 2, Bytes.toBytes("Data23")); - t.put(p); - String[] args = { sourceTable, outputDir.toString(), ";" }; - runCount(args); - FileInputStream inputStream = - new FileInputStream(outputPath + File.separator + "part-r-00000"); - String data = IOUtils.toString(inputStream); - inputStream.close(); - assertTrue(data.contains("Total Families Across all Rows" + "\t" + "2")); - assertTrue(data.contains("Total Qualifiers across all Rows" + "\t" + "4")); - assertTrue(data.contains("Total ROWS" + "\t" + "2")); - assertTrue(data.contains("b;q" + "\t" + "2")); - assertTrue(data.contains("a;q" + "\t" + "2")); - assertTrue(data.contains("row1;a;q_Versions" + "\t" + "1")); - assertTrue(data.contains("row1;b;q_Versions" + "\t" + "1")); - assertTrue(data.contains("row2;a;q_Versions" + "\t" + "1")); - assertTrue(data.contains("row2;b;q_Versions" + "\t" + "1")); - } finally { - t.close(); - FileUtil.fullyDelete(new File(outputPath)); - } - } - - @Test - public void TestCellCounterWithoutOutputDir() throws Exception { - String[] args = new String[] { "tableName" }; - assertEquals("CellCounter should exit with -1 as output directory is not specified.", -1, - ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args)); - } } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index bf4eb0997b19..9bdebe6fa903 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -795,7 +795,7 @@ public void testColumnFamilySettings() throws Exception { HTable table = Mockito.mock(HTable.class); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); Mockito.doReturn(htd).when(table).getTableDescriptor(); - for (HColumnDescriptor hcd: HBaseTestingUtility.generateColumnDescriptors()) { + for (HColumnDescriptor hcd: this.util.generateColumnDescriptors()) { htd.addFamily(hcd); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java index a9a75c86301d..d3019ce210a4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java @@ -432,33 +432,5 @@ public void testWithoutAnExistingTableAndCreateTableSetToNo() throws Exception { String[] args = { "directory", "nonExistingTable" }; loader.run(args); } - - @Test - public void testTableWithCFNameStartWithUnderScore() throws Exception { - Path dir = util.getDataTestDirOnTestFS("cfNameStartWithUnderScore"); - FileSystem fs = util.getTestFileSystem(); - dir = dir.makeQualified(fs.getUri(), fs.getWorkingDirectory()); - String family = "_cf"; - Path familyDir = new Path(dir, family); - - byte[] from = Bytes.toBytes("begin"); - byte[] to = Bytes.toBytes("end"); - Configuration conf = util.getConfiguration(); - String tableName = "mytable_cfNameStartWithUnderScore"; - Table table = util.createTable(TableName.valueOf(tableName), family); - HFileTestUtil.createHFile(conf, fs, new Path(familyDir, "hfile"), Bytes.toBytes(family), - QUALIFIER, from, to, 1000); - - LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf); - String[] args = { dir.toString(), tableName }; - try { - loader.run(args); - assertEquals(1000, util.countRows(table)); - } finally { - if (null != table) { - table.close(); - } - } - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java index 47501d8c8574..5c4ae776c60f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java @@ -151,7 +151,7 @@ private void setupTableWithSplitkeys(TableName table, int cfs, byte[][] SPLIT_KE htd.addFamily(new HColumnDescriptor(family(i))); } - util.createTable(htd, SPLIT_KEYS); + util.getHBaseAdmin().createTable(htd, SPLIT_KEYS); } catch (TableExistsException tee) { LOG.info("Table " + table + " already exists"); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index 8d7e2d38e105..903e0c0667d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -77,42 +77,37 @@ public void testGetBestLocations() throws IOException { Configuration conf = UTIL.getConfiguration(); HDFSBlocksDistribution blockDistribution = new HDFSBlocksDistribution(); - Assert.assertEquals(Lists.newArrayList(), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + Assert.assertEquals(Lists.newArrayList(), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1); - Assert.assertEquals(Lists.newArrayList("h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 1); - Assert.assertEquals(Lists.newArrayList("h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 1); - Assert.assertEquals(Lists.newArrayList("h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution = new HDFSBlocksDistribution(); blockDistribution.addHostsAndBlockWeight(new String[] {"h1"}, 10); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 7); blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 5); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1); - Assert.assertEquals(Lists.newArrayList("h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + Assert.assertEquals(Lists.newArrayList("h1"), tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 2); Assert.assertEquals(Lists.newArrayList("h1", "h2"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h2"}, 3); Assert.assertEquals(Lists.newArrayList("h2", "h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + tsif.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h3"}, 6); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 9); Assert.assertEquals(Lists.newArrayList("h2", "h3", "h4", "h1"), - TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); + tsif.getBestLocations(conf, blockDistribution)); } public static enum TestTableSnapshotCounters { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestJarFinder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java similarity index 96% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestJarFinder.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java index 8187b73aa463..944475e0a3c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestJarFinder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/hadoopbackport/TestJarFinder.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.hadoop.hbase.mapreduce; +package org.apache.hadoop.hbase.mapreduce.hadoopbackport; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Assert; import org.junit.Test; @@ -41,7 +42,7 @@ /** * This file was forked from hadoop/common/branches/branch-2@1350012. */ -@Category(SmallTests.class) +@Category({MapReduceTests.class, SmallTests.class}) public class TestJarFinder { @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index 9bf3d1020536..e084043d5ba8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -348,14 +348,6 @@ public void add(HTableDescriptor htd) throws IOException { public void add(TableDescriptor htd) throws IOException { // noop } - - @Override - public void setCacheOn() throws IOException { - } - - @Override - public void setCacheOff() throws IOException { - } }; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index 5321ae8bdefe..e936acefaa72 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -119,9 +119,7 @@ public class TestDistributedLogSplitting { private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); static { - // Uncomment the following line if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); // test ThreeRSAbort fails under hadoop2 (2.0.2-alpha) if shortcircuit-read (scr) is on. this // turns it off for this test. TODO: Figure out why scr breaks recovery. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java index 76a6db974e1e..5847b9347ca9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java @@ -203,7 +203,7 @@ public void setup() throws Exception { Mockito.when(sm.isServerOnline(Mockito.any(ServerName.class))).thenReturn(true); Mockito.when(master.getServerManager()).thenReturn(sm); - to = 12000; + to = 6000; conf.setInt(HConstants.HBASE_SPLITLOG_MANAGER_TIMEOUT, to); conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java index 2bb8076c46ab..64666abf7201 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksRead.java @@ -48,8 +48,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -77,13 +75,13 @@ private HBaseConfiguration getConf() { * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() */ @SuppressWarnings("deprecation") - @Before + @Override protected void setUp() throws Exception { super.setUp(); } @SuppressWarnings("deprecation") - @After + @Override protected void tearDown() throws Exception { super.tearDown(); EnvironmentEdgeManagerTestHelper.reset(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java index 25330a88d9d7..0833a6a4f762 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -50,7 +49,7 @@ public class TestBlocksScanned extends HBaseTestCase { private static HBaseTestingUtility TEST_UTIL = null; - @Before + @Override public void setUp() throws Exception { super.setUp(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index d6d82df6e71c..b07471c2291a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.KeyValueUtil; @@ -94,9 +93,8 @@ public void testScanAcrossSnapshot() throws IOException { List memstorescanners = this.memstore.getScanners(0); Scan scan = new Scan(); List result = new ArrayList(); - ScanInfo scanInfo = - new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, KeepDeletedCells.FALSE, 0, - this.memstore.comparator); + ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false, + 0, this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners); int count = 0; @@ -515,7 +513,7 @@ public void testGetNextRow() throws Exception { } //starting from each row, validate results should contain the starting row for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) { - ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE, + ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false, 0, this.memstore.comparator); ScanType scanType = ScanType.USER_SCAN; InternalScanner scanner = new StoreScanner(new Scan( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 962b408aa0db..1651ab64fd56 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -385,7 +385,7 @@ protected void chore() { verify(); } catch (Throwable ex) { this.ex = ex; - getStopper().stop("caught exception"); + stopper.stop("caught exception"); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java index b270a7db6606..988d40bd89b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -66,7 +65,6 @@ public class TestGetClosestAtOrBefore extends HBaseTestCase { - @Test public void testUsingMetaAndBinary() throws IOException { FileSystem filesystem = FileSystem.get(conf); Path rootdir = testDir; @@ -190,7 +188,6 @@ public void testUsingMetaAndBinary() throws IOException { * Test file of multiple deletes and with deletes as final key. * @see HBASE-751 */ - @Test public void testGetClosestRowBefore3() throws IOException{ HRegion region = null; byte [] c0 = COLUMNS[0]; @@ -299,7 +296,6 @@ public void testGetClosestRowBefore3() throws IOException{ } /** For HBASE-694 */ - @Test public void testGetClosestRowBefore2() throws IOException{ HRegion region = null; byte [] c0 = COLUMNS[0]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java index 341b02c61497..94690962203e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeepDeletes.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.client.Delete; @@ -97,7 +96,7 @@ public void tearDown() throws Exception { public void testBasicScenario() throws Exception { // keep 3 versions, rows do not expire HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -194,7 +193,7 @@ public void testBasicScenario() throws Exception { public void testRawScanWithoutKeepingDeletes() throws Exception { // KEEP_DELETED_CELLS is NOT enabled HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.FALSE); + HConstants.FOREVER, false); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -239,7 +238,7 @@ public void testRawScanWithoutKeepingDeletes() throws Exception { public void testWithoutKeepingDeletes() throws Exception { // KEEP_DELETED_CELLS is NOT enabled HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.FALSE); + HConstants.FOREVER, false); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -283,7 +282,7 @@ public void testWithoutKeepingDeletes() throws Exception { @Test public void testRawScanWithColumns() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); Scan s = new Scan(); @@ -307,7 +306,7 @@ public void testRawScanWithColumns() throws Exception { @Test public void testRawScan() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -397,7 +396,7 @@ public void testRawScan() throws Exception { @Test public void testDeleteMarkerExpirationEmptyStore() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -440,7 +439,7 @@ public void testDeleteMarkerExpirationEmptyStore() throws Exception { @Test public void testDeleteMarkerExpiration() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -497,91 +496,13 @@ public void testDeleteMarkerExpiration() throws Exception { HRegion.closeHRegion(region); } - /** - * Test delete marker removal from store files. - */ - @Test - public void testWithOldRow() throws Exception { - HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, KeepDeletedCells.TRUE); - HRegion region = hbu.createLocalHRegion(htd, null, null); - - long ts = EnvironmentEdgeManager.currentTime(); - - Put p = new Put(T1, ts); - p.add(c0, c0, T1); - region.put(p); - - // a put another (older) row in the same store - p = new Put(T2, ts-10); - p.add(c0, c0, T1); - region.put(p); - - // all the following deletes affect the put - Delete d = new Delete(T1, ts); - d.deleteColumns(c0, c0, ts); - region.delete(d); - - d = new Delete(T1, ts); - d.deleteFamily(c0, ts); - region.delete(d); - - d = new Delete(T1, ts); - d.deleteColumn(c0, c0, ts+1); - region.delete(d); - - d = new Delete(T1, ts); - d.deleteColumn(c0, c0, ts+2); - region.delete(d); - - // 1 family marker, 1 column marker, 2 version markers - assertEquals(4, countDeleteMarkers(region)); - - region.flushcache(); - assertEquals(4, countDeleteMarkers(region)); - region.compactStores(false); - assertEquals(4, countDeleteMarkers(region)); - - // another put will push out the earlier put... - p = new Put(T1, ts+3); - p.add(c0, c0, T1); - region.put(p); - - region.flushcache(); - // no markers are collected, since there is an affected put - region.compactStores(true); - assertEquals(4, countDeleteMarkers(region)); - - // all markers remain, since we have the older row - // and we haven't pushed the inlined markers past MAX_VERSIONS - region.compactStores(true); - assertEquals(4, countDeleteMarkers(region)); - - // another put will push out the earlier put... - p = new Put(T1, ts+4); - p.add(c0, c0, T1); - region.put(p); - - // this pushed out the column and version marker - // but the family markers remains. THIS IS A PROBLEM! - region.compactStores(true); - assertEquals(1, countDeleteMarkers(region)); - - // no amount of compacting is getting this of this one - // KEEP_DELETED_CELLS=>TTL is an option to avoid this. - region.compactStores(true); - assertEquals(1, countDeleteMarkers(region)); - - HRegion.closeHRegion(region); - } - /** * Verify correct range demarcation */ @Test public void testRanges() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 3, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -663,7 +584,7 @@ public void testRanges() throws Exception { @Test public void testDeleteMarkerVersioning() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -755,7 +676,7 @@ public void testDeleteMarkerVersioning() throws Exception { */ public void testWithMixedCFs() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 0, 1, - HConstants.FOREVER, KeepDeletedCells.TRUE); + HConstants.FOREVER, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime(); @@ -806,8 +727,7 @@ public void testWithMixedCFs() throws Exception { */ @Test public void testWithMinVersions() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.TRUE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, true); HRegion region = hbu.createLocalHRegion(htd, null, null); long ts = EnvironmentEdgeManager.currentTime() - 2000; // 2s in the past @@ -879,51 +799,6 @@ public void testWithMinVersions() throws Exception { HRegion.closeHRegion(region); } - /** - * Test keeping deleted rows together with min versions set - * @throws Exception - */ - @Test - public void testWithTTL() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.TTL); - HRegion region = hbu.createLocalHRegion(htd, null, null); - - long ts = EnvironmentEdgeManager.currentTime() - 2000; // 2s in the past - - Put p = new Put(T1, ts); - p.add(c0, c0, T3); - region.put(p); - - // place an old row, to make the family marker expires anyway - p = new Put(T2, ts-10); - p.add(c0, c0, T1); - region.put(p); - - checkGet(region, T1, c0, c0, ts+1, T3); - // place a family delete marker - Delete d = new Delete(T1, ts+2); - region.delete(d); - - checkGet(region, T1, c0, c0, ts+1, T3); - - // 3 families, one column delete marker - assertEquals(3, countDeleteMarkers(region)); - - region.flushcache(); - // no delete marker removes by the flush - assertEquals(3, countDeleteMarkers(region)); - - // but the Put is gone - checkGet(region, T1, c0, c0, ts+1); - - region.compactStores(true); - // all delete marker gone - assertEquals(0, countDeleteMarkers(region)); - - HRegion.closeHRegion(region); - } - private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col, long time, byte[]... vals) throws IOException { Get g = new Get(row); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java index 86a15ff052fa..b1aed0b7c288 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java @@ -31,8 +31,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CollectionBackedScanner; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, SmallTests.class}) @@ -54,7 +52,6 @@ public class TestKeyValueHeap extends HBaseTestCase { private byte[] col4; private byte[] col5; - @Before public void setUp() throws Exception { super.setUp(); data = Bytes.toBytes("data"); @@ -69,7 +66,6 @@ public void setUp() throws Exception { col5 = Bytes.toBytes("col5"); } - @Test public void testSorted() throws IOException{ //Cases that need to be checked are: //1. The "smallest" KeyValue is in the same scanners as current @@ -132,7 +128,6 @@ public void testSorted() throws IOException{ } - @Test public void testSeek() throws IOException { //Cases: //1. Seek KeyValue that is not in scanner @@ -181,7 +176,6 @@ public void testSeek() throws IOException { } - @Test public void testScannerLeak() throws IOException { // Test for unclosed scanners (HBASE-1927) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java index d022acbfb940..8c9da0bab927 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.client.Delete; @@ -66,8 +65,7 @@ public class TestMinVersions { */ @Test public void testGetClosestBefore() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -116,8 +114,7 @@ public void testGetClosestBefore() throws Exception { @Test public void testStoreMemStore() throws Exception { // keep 3 versions minimum - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past long ts = EnvironmentEdgeManager.currentTime() - 2000; @@ -171,8 +168,7 @@ public void testStoreMemStore() throws Exception { */ @Test public void testDelete() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -230,8 +226,7 @@ public void testDelete() throws Exception { */ @Test public void testMemStore() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); // 2s in the past @@ -306,8 +301,7 @@ public void testMemStore() throws Exception { @Test public void testBaseCase() throws Exception { // 1 version minimum, 1000 versions maximum, ttl = 1s - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); try { @@ -398,8 +392,7 @@ public void testBaseCase() throws Exception { */ @Test public void testFilters() throws Exception { - HTableDescriptor htd = - hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); + HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, false); HRegion region = hbu.createLocalHRegion(htd, null, null); final byte [] c1 = COLUMNS[1]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java index 97c66e49e517..2b1f11fd4006 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.Type; @@ -40,8 +39,6 @@ import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, SmallTests.class}) @@ -67,7 +64,6 @@ public class TestQueryMatcher extends HBaseTestCase { KVComparator rowComparator; private Scan scan; - @Before public void setUp() throws Exception { super.setUp(); row1 = Bytes.toBytes("row1"); @@ -98,7 +94,7 @@ public void setUp() throws Exception { private void _testMatch_ExplicitColumns(Scan scan, List expected) throws IOException { // 2,4,5 ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), get.getFamilyMap().get(fam2), + 0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2), EnvironmentEdgeManager.currentTime() - ttl); List memstore = new ArrayList(); @@ -128,7 +124,6 @@ private void _testMatch_ExplicitColumns(Scan scan, List expected) thr } } - @Test public void testMatch_ExplicitColumns() throws IOException { //Moving up from the Tracker by using Gets and List instead @@ -146,7 +141,6 @@ public void testMatch_ExplicitColumns() _testMatch_ExplicitColumns(scan, expected); } - @Test public void testMatch_ExplicitColumnsWithLookAhead() throws IOException { //Moving up from the Tracker by using Gets and List instead @@ -167,7 +161,6 @@ public void testMatch_ExplicitColumnsWithLookAhead() } - @Test public void testMatch_Wildcard() throws IOException { //Moving up from the Tracker by using Gets and List instead @@ -183,7 +176,7 @@ public void testMatch_Wildcard() expected.add(ScanQueryMatcher.MatchCode.DONE); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator), null, + 0, 1, ttl, false, 0, rowComparator), null, EnvironmentEdgeManager.currentTime() - ttl); List memstore = new ArrayList(); @@ -222,7 +215,6 @@ public void testMatch_Wildcard() * * @throws IOException */ - @Test public void testMatch_ExpiredExplicit() throws IOException { @@ -237,9 +229,9 @@ public void testMatch_ExpiredExplicit() }; long now = EnvironmentEdgeManager.currentTime(); - ScanQueryMatcher qm = - new ScanQueryMatcher(scan, new ScanInfo(fam2, 0, 1, testTTL, KeepDeletedCells.FALSE, 0, - rowComparator), get.getFamilyMap().get(fam2), now - testTTL); + ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, + 0, 1, testTTL, false, 0, rowComparator), get.getFamilyMap().get(fam2), + now - testTTL); KeyValue [] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now-100, data), @@ -277,7 +269,6 @@ public void testMatch_ExpiredExplicit() * * @throws IOException */ - @Test public void testMatch_ExpiredWildcard() throws IOException { @@ -293,7 +284,7 @@ public void testMatch_ExpiredWildcard() long now = EnvironmentEdgeManager.currentTime(); ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2, - 0, 1, testTTL, KeepDeletedCells.FALSE, 0, rowComparator), null, + 0, 1, testTTL, false, 0, rowComparator), null, now - testTTL); KeyValue [] kvs = new KeyValue[] { @@ -323,7 +314,6 @@ public void testMatch_ExpiredWildcard() } } - @Test public void testMatch_PartialRangeDropDeletes() throws Exception { // Some ranges. testDropDeletes( @@ -349,7 +339,7 @@ private void testDropDeletes( byte[] from, byte[] to, byte[][] rows, MatchCode... expected) throws IOException { long now = EnvironmentEdgeManager.currentTime(); // Set time to purge deletes to negative value to avoid it ever happening. - ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, KeepDeletedCells.FALSE, -1L, rowComparator); + ScanInfo scanInfo = new ScanInfo(fam2, 0, 1, ttl, false, -1L, rowComparator); NavigableSet cols = get.getFamilyMap().get(fam2); ScanQueryMatcher qm = new ScanQueryMatcher(scan, scanInfo, cols, Long.MAX_VALUE, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java index cbe79fe5552e..6e4030b32fa7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java @@ -29,7 +29,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -43,13 +42,11 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.util.StringUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -79,12 +76,6 @@ public class TestRegionReplicas { @BeforeClass public static void before() throws Exception { - // Reduce the hdfs block size and prefetch to trigger the file-link reopen - // when the file is moved to archive (e.g. compaction) - HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192); - HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1); - HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024); - HTU.startMiniCluster(NB_SERVERS); final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName()); @@ -408,67 +399,9 @@ public void run() { for (AtomicReference exRef : exceptions) { Assert.assertNull(exRef.get()); } - } finally { - HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey); - closeRegion(HTU, getRS(), hriSecondary); - } - } - - @Test(timeout = 300000) - public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception { - // disable the store file refresh chore (we do this by hand) - HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0); - restartRegionServer(); - - try { - LOG.info("Opening the secondary region " + hriSecondary.getEncodedName()); - openRegion(HTU, getRS(), hriSecondary); - - // load some data to primary - LOG.info("Loading data to primary region"); - for (int i = 0; i < 3; ++i) { - HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000); - getRS().getRegionByEncodedName(hriPrimary.getEncodedName()).flushcache(); - } - - HRegion primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName()); - Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount()); - - // Refresh store files on the secondary - HRegion secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName()); - secondaryRegion.getStore(f).refreshStoreFiles(); - Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount()); - // force compaction - LOG.info("Force Major compaction on primary region " + hriPrimary); - primaryRegion.compactStores(true); - Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount()); - - // scan all the hfiles on the secondary. - // since there are no read on the secondary when we ask locations to - // the NN a FileNotFound exception will be returned and the FileLink - // should be able to deal with it giving us all the result we expect. - int keys = 0; - int sum = 0; - for (StoreFile sf: secondaryRegion.getStore(f).getStorefiles()) { - // Our file does not exist anymore. was moved by the compaction above. - LOG.debug(getRS().getFileSystem().exists(sf.getPath())); - Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath())); - - HFileScanner scanner = sf.getReader().getScanner(false, false); - scanner.seekTo(); - do { - keys++; - - Cell cell = scanner.getKeyValue(); - sum += Integer.parseInt(Bytes.toString(cell.getRowArray(), - cell.getRowOffset(), cell.getRowLength())); - } while (scanner.next()); - } - Assert.assertEquals(3000, keys); - Assert.assertEquals(4498500, sum); } finally { - HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000); + HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey); closeRegion(HTU, getRS(), hriSecondary); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java index 65aed5bae004..24d9dd459378 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java @@ -212,6 +212,40 @@ public void testCloseByRegionServer() throws Exception { openRegion(HTU, getRS(), hri); } + /** + * Test that we can send multiple openRegion to the region server. + * This is used when: + * - there is a SocketTimeout: in this case, the master does not know if the region server + * received the request before the timeout. + * - We have a socket error during the operation: same stuff: we don't know + * - a master failover: if we find a znode in thz M_ZK_REGION_OFFLINE, we don't know if + * the region server has received the query or not. Only solution to be efficient: re-ask + * immediately. + */ + @Test(timeout = 60000) + public void testMultipleOpen() throws Exception { + + // We close + closeRegionNoZK(); + checkRegionIsClosed(HTU, getRS(), hri); + + // We're sending multiple requests in a row. The region server must handle this nicely. + for (int i = 0; i < 10; i++) { + AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest( + getRS().getServerName(), hri, null, null); + AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr); + Assert.assertTrue(responseOpen.getOpeningStateCount() == 1); + + AdminProtos.OpenRegionResponse.RegionOpeningState ors = responseOpen.getOpeningState(0); + Assert.assertTrue("request " + i + " failed", + ors.equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED) || + ors.equals(AdminProtos.OpenRegionResponse.RegionOpeningState.ALREADY_OPENED) + ); + } + + checkRegionIsOpened(HTU, getRS(), hri); + } + @Test(timeout = 60000) public void testMultipleCloseFromMaster() throws Exception { for (int i = 0; i < 10; i++) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java index 5a95df11ade4..197c3809ee33 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -253,7 +252,7 @@ public void testReversibleStoreScanner() throws IOException { ScanType scanType = ScanType.USER_SCAN; ScanInfo scanInfo = new ScanInfo(FAMILYNAME, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, false, 0, KeyValue.COMPARATOR); // Case 1.Test a full reversed scan Scan scan = new Scan(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java index 5bc77b572b62..537ab9592430 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java @@ -73,11 +73,7 @@ public void testScannersSeekOnFewLargeCells() throws IOException { HTableDescriptor htd = TEST_HTD; HColumnDescriptor hcd = new HColumnDescriptor(fam1); - if (htd.hasFamily(hcd.getName())) { - htd.modifyFamily(hcd); - } else { - htd.addFamily(hcd); - } + htd.addFamily(hcd); final HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, @@ -115,11 +111,7 @@ public void testScanAcrossManySmallColumns() throws IOException { HTableDescriptor htd = TEST_HTD; HColumnDescriptor hcd = new HColumnDescriptor(fam1); - if (htd.hasFamily(hcd.getName())) { - htd.modifyFamily(hcd); - } else { - htd.addFamily(hcd); - } + htd.addFamily(hcd); final HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java index 2854832f88fa..f2fb44349eeb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java @@ -25,8 +25,6 @@ import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, SmallTests.class}) @@ -36,13 +34,11 @@ public class TestScanDeleteTracker extends HBaseTestCase { private long timestamp = 10L; private byte deleteType = 0; - @Before public void setUp() throws Exception { super.setUp(); sdt = new ScanDeleteTracker(); } - @Test public void testDeletedBy_Delete() { KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("f"), Bytes.toBytes("qualifier"), timestamp, KeyValue.Type.Delete); @@ -51,7 +47,6 @@ public void testDeletedBy_Delete() { assertEquals(DeleteResult.VERSION_DELETED, ret); } - @Test public void testDeletedBy_DeleteColumn() { KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("f"), Bytes.toBytes("qualifier"), timestamp, KeyValue.Type.DeleteColumn); @@ -63,7 +58,6 @@ public void testDeletedBy_DeleteColumn() { assertEquals(DeleteResult.COLUMN_DELETED, ret); } - @Test public void testDeletedBy_DeleteFamily() { KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("f"), Bytes.toBytes("qualifier"), timestamp, KeyValue.Type.DeleteFamily); @@ -75,7 +69,6 @@ public void testDeletedBy_DeleteFamily() { assertEquals(DeleteResult.FAMILY_DELETED, ret); } - @Test public void testDeletedBy_DeleteFamilyVersion() { byte [] qualifier1 = Bytes.toBytes("qualifier1"); byte [] qualifier2 = Bytes.toBytes("qualifier2"); @@ -120,7 +113,6 @@ public void testDeletedBy_DeleteFamilyVersion() { } - @Test public void testDelete_DeleteColumn() { byte [] qualifier = Bytes.toBytes("qualifier"); deleteType = KeyValue.Type.Delete.getCode(); @@ -142,7 +134,6 @@ public void testDelete_DeleteColumn() { } - @Test public void testDeleteColumn_Delete() { byte [] qualifier = Bytes.toBytes("qualifier"); deleteType = KeyValue.Type.DeleteColumn.getCode(); @@ -163,7 +154,6 @@ public void testDeleteColumn_Delete() { //Testing new way where we save the Delete in case of a Delete for specific //ts, could have just added the last line to the first test, but rather keep //them separated - @Test public void testDelete_KeepDelete(){ byte [] qualifier = Bytes.toBytes("qualifier"); deleteType = KeyValue.Type.Delete.getCode(); @@ -174,7 +164,6 @@ public void testDelete_KeepDelete(){ assertEquals(false ,sdt.isEmpty()); } - @Test public void testDelete_KeepVersionZero(){ byte [] qualifier = Bytes.toBytes("qualifier"); deleteType = KeyValue.Type.Delete.getCode(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java index daee137aa2fc..c0dcee6a34b8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, SmallTests.class}) @@ -36,7 +35,6 @@ public class TestScanWildcardColumnTracker extends HBaseTestCase { final static int VERSIONS = 2; - @Test public void testCheckColumn_Ok() throws IOException { ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(0, VERSIONS, Long.MIN_VALUE); @@ -70,7 +68,6 @@ public void testCheckColumn_Ok() throws IOException { } } - @Test public void testCheckColumn_EnforceVersions() throws IOException { ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(0, VERSIONS, Long.MIN_VALUE); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 39b41dd848de..f4b70ed277e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -175,11 +175,7 @@ private Store init(String methodName, Configuration conf, HTableDescriptor htd, fs.delete(logdir, true); - if (htd.hasFamily(hcd.getName())) { - htd.modifyFamily(hcd); - } else { - htd.addFamily(hcd); - } + htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf); HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 95f66964fdc1..04e35165c436 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -58,9 +58,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -81,12 +78,12 @@ public class TestStoreFile extends HBaseTestCase { private static final int CKBYTES = 512; private static String TEST_FAMILY = "cf"; - @Before + @Override public void setUp() throws Exception { super.setUp(); } - @After + @Override public void tearDown() throws Exception { super.tearDown(); } @@ -96,12 +93,11 @@ public void tearDown() throws Exception { * using two HalfMapFiles. * @throws Exception */ - @Test public void testBasicHalfMapFile() throws Exception { final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testBasicHalfMapFileTb")); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); + conf, fs, new Path(this.testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(2*1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, this.fs) @@ -149,11 +145,10 @@ public static void writeStoreFile(final StoreFile.Writer writer, byte[] fam, byt * store files in other regions works. * @throws IOException */ - @Test public void testReference() throws IOException { final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testReferenceTb")); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); + conf, fs, new Path(this.testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. @@ -193,14 +188,13 @@ public void testReference() throws IOException { assertTrue(Bytes.equals(kv.getRow(), finalRow)); } - @Test public void testHFileLink() throws IOException { final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testHFileLinkTb")); // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ Configuration testConf = new Configuration(this.conf); - FSUtils.setRootDir(testConf, testDir); + FSUtils.setRootDir(testConf, this.testDir); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri); + testConf, fs, FSUtils.getTableDir(this.testDir, hri.getTable()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. @@ -236,16 +230,15 @@ public void testHFileLink() throws IOException { * This test creates an hfile and then the dir structures and files to verify that references * to hfilelinks (created by snapshot clones) can be properly interpreted. */ - @Test public void testReferenceToHFileLink() throws IOException { // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ Configuration testConf = new Configuration(this.conf); - FSUtils.setRootDir(testConf, testDir); + FSUtils.setRootDir(testConf, this.testDir); // adding legal table name chars to verify regex handles it. HRegionInfo hri = new HRegionInfo(TableName.valueOf("_original-evil-name")); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri); + testConf, fs, FSUtils.getTableDir(this.testDir, hri.getTable()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. //// @@ -259,7 +252,7 @@ public void testReferenceToHFileLink() throws IOException { // create link to store file. /clone/region//-- HRegionInfo hriClone = new HRegionInfo(TableName.valueOf("clone")); HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem( - testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), + testConf, fs, FSUtils.getTableDir(this.testDir, hri.getTable()), hriClone); Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); @@ -276,7 +269,7 @@ public void testReferenceToHFileLink() throws IOException { Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom // OK test the thing - FSUtils.logFileSystemState(fs, testDir, LOG); + FSUtils.logFileSystemState(fs, this.testDir, LOG); // There is a case where a file with the hfilelink pattern is actually a daughter // reference to a hfile link. This code in StoreFile that handles this case. @@ -501,7 +494,6 @@ private void bloomWriteRead(StoreFile.Writer writer, FileSystem fs) throws Excep private static final int BLOCKSIZE_SMALL = 8192; - @Test public void testBloomFilter() throws Exception { FileSystem fs = FileSystem.getLocal(conf); conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); @@ -522,7 +514,6 @@ public void testBloomFilter() throws Exception { bloomWriteRead(writer, fs); } - @Test public void testDeleteFamilyBloomFilter() throws Exception { FileSystem fs = FileSystem.getLocal(conf); conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); @@ -585,7 +576,6 @@ public void testDeleteFamilyBloomFilter() throws Exception { /** * Test for HBASE-8012 */ - @Test public void testReseek() throws Exception { // write the file Path f = new Path(ROOT_DIR, getName()); @@ -610,7 +600,6 @@ public void testReseek() throws Exception { assertNotNull("Intial reseek should position at the beginning of the file", s.peek()); } - @Test public void testBloomTypes() throws Exception { float err = (float) 0.01; FileSystem fs = FileSystem.getLocal(conf); @@ -699,7 +688,6 @@ public void testBloomTypes() throws Exception { } } - @Test public void testSeqIdComparator() { assertOrdering(StoreFile.Comparators.SEQ_ID, mockStoreFile(true, 100, 1000, -1, "/foo/123"), @@ -778,7 +766,6 @@ List getKeyValueSet(long[] timestamps, int numRows, * Test to ensure correctness when using StoreFile with multiple timestamps * @throws IOException */ - @Test public void testMultipleTimestamps() throws IOException { byte[] family = Bytes.toBytes("familyname"); byte[] qualifier = Bytes.toBytes("qualifier"); @@ -787,7 +774,7 @@ public void testMultipleTimestamps() throws IOException { Scan scan = new Scan(); // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. - Path storedir = new Path(new Path(testDir, "7e0102"), "familyname"); + Path storedir = new Path(new Path(this.testDir, "7e0102"), "familyname"); Path dir = new Path(storedir, "1234567890"); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. @@ -829,12 +816,11 @@ public void testMultipleTimestamps() throws IOException { assertTrue(!scanner.shouldUseScanner(scan, columns, Long.MIN_VALUE)); } - @Test public void testCacheOnWriteEvictOnClose() throws Exception { Configuration conf = this.conf; // Find a home for our files (regiondir ("7e0102") and familyname). - Path baseDir = new Path(new Path(testDir, "7e0102"),"twoCOWEOC"); + Path baseDir = new Path(new Path(this.testDir, "7e0102"),"twoCOWEOC"); // Grab the block cache and get the initial hit/miss counts BlockCache bc = new CacheConfig(conf).getBlockCache(); @@ -1002,10 +988,9 @@ private StoreFile.Writer writeStoreFile(Configuration conf, * Check if data block encoding information is saved correctly in HFile's * file info. */ - @Test public void testDataBlockEncodingMetaData() throws IOException { // Make up a directory hierarchy that has a regiondir ("7e0102") and familyname. - Path dir = new Path(new Path(testDir, "7e0102"), "familyname"); + Path dir = new Path(new Path(this.testDir, "7e0102"), "familyname"); Path path = new Path(dir, "1234567890"); DataBlockEncoding dataBlockEncoderAlgo = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java index da39f5955b99..4bb19d5bda52 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFileInfo.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.io.HFileLink; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -36,7 +35,6 @@ public class TestStoreFileInfo extends HBaseTestCase { /** * Validate that we can handle valid tables with '.', '_', and '-' chars. */ - @Test public void testStoreFileNames() { String[] legalHFileLink = { "MyTable_02=abc012-def345", "MyTable_02.300=abc012-def345", "MyTable_02-400=abc012-def345", "MyTable_02-400.200=abc012-def345", diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index bf9fed69f0c3..320ed567f4e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -49,7 +48,7 @@ public class TestStoreScanner extends TestCase { private static final String CF_STR = "cf"; final byte [] CF = Bytes.toBytes(CF_STR); private ScanInfo scanInfo = new ScanInfo(CF, 0, Integer.MAX_VALUE, - Long.MAX_VALUE, KeepDeletedCells.FALSE, 0, KeyValue.COMPARATOR); + Long.MAX_VALUE, false, 0, KeyValue.COMPARATOR); private ScanType scanType = ScanType.USER_SCAN; public void setUp() throws Exception { @@ -417,7 +416,7 @@ public void testWildCardTtlScan() throws IOException { List scanners = scanFixture(kvs); Scan scan = new Scan(); scan.setMaxVersions(1); - ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = @@ -488,7 +487,7 @@ public void testExpiredDeleteFamily() throws Exception { Scan scan = new Scan(); scan.setMaxVersions(1); // scanner with ttl equal to 500 - ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, KeepDeletedCells.FALSE, 0, + ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = @@ -551,7 +550,7 @@ public long currentTime() { ScanInfo scanInfo = new ScanInfo(Bytes.toBytes("cf"), 0 /* minVersions */, 2 /* maxVersions */, 500 /* ttl */, - KeepDeletedCells.FALSE /* keepDeletedCells */, + false /* keepDeletedCells */, 200, /* timeToPurgeDeletes */ KeyValue.COMPARATOR); StoreScanner scanner = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java index 1c8f405cf6a0..a6133195e30a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java @@ -125,12 +125,13 @@ public void testTags() throws Exception { put.setAttribute("visibility", Bytes.toBytes("myTag")); table.put(put); admin.flush(tableName); - // We are lacking an API for confirming flush request compaction. - // Just sleep for a short time. We won't be able to confirm flush - // completion but the test won't hang now or in the future if - // default compaction policy causes compaction between flush and - // when we go to confirm it. - Thread.sleep(1000); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 0)) { + Thread.sleep(10); + } + } Put put1 = new Put(row1); byte[] value1 = Bytes.toBytes("1000dfsdf"); @@ -138,18 +139,29 @@ public void testTags() throws Exception { // put1.setAttribute("visibility", Bytes.toBytes("myTag3")); table.put(put1); admin.flush(tableName); - Thread.sleep(1000); + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 1)) { + Thread.sleep(10); + } + } Put put2 = new Put(row2); byte[] value2 = Bytes.toBytes("1000dfsdf"); put2.add(fam, qual, HConstants.LATEST_TIMESTAMP, value2); put2.setAttribute("visibility", Bytes.toBytes("myTag3")); table.put(put2); - admin.flush(tableName); - Thread.sleep(1000); + admin.flush(tableName); + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName.getName()); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 2)) { + Thread.sleep(10); + } + } result(fam, row, qual, row2, table, value, value2, row1, value1); - admin.compact(tableName); while (admin.getCompactionState(tableName) != CompactionState.NONE) { Thread.sleep(10); @@ -191,27 +203,40 @@ public void testFlushAndCompactionWithoutTags() throws Exception { put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value); table.put(put); admin.flush(tableName); - // We are lacking an API for confirming flush request compaction. - // Just sleep for a short time. We won't be able to confirm flush - // completion but the test won't hang now or in the future if - // default compaction policy causes compaction between flush and - // when we go to confirm it. - Thread.sleep(1000); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName.getName()); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 0)) { + Thread.sleep(10); + } + } Put put1 = new Put(row1); byte[] value1 = Bytes.toBytes("1000dfsdf"); put1.add(fam, qual, HConstants.LATEST_TIMESTAMP, value1); table.put(put1); admin.flush(tableName); - Thread.sleep(1000); + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName.getName()); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 1)) { + Thread.sleep(10); + } + } Put put2 = new Put(row2); byte[] value2 = Bytes.toBytes("1000dfsdf"); put2.add(fam, qual, HConstants.LATEST_TIMESTAMP, value2); table.put(put2); - admin.flush(tableName); - Thread.sleep(1000); + admin.flush(tableName); + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 2)) { + Thread.sleep(10); + } + } Scan s = new Scan(row); ResultScanner scanner = table.getScanner(s); try { @@ -287,20 +312,26 @@ public void testFlushAndCompactionwithCombinations() throws Exception { put1.add(fam, qual, HConstants.LATEST_TIMESTAMP, value1); table.put(put1); admin.flush(tableName); - // We are lacking an API for confirming flush request compaction. - // Just sleep for a short time. We won't be able to confirm flush - // completion but the test won't hang now or in the future if - // default compaction policy causes compaction between flush and - // when we go to confirm it. - Thread.sleep(1000); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName.getName()); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 0)) { + Thread.sleep(10); + } + } put1 = new Put(row2); value1 = Bytes.toBytes("1000dfsdf"); put1.add(fam, qual, HConstants.LATEST_TIMESTAMP, value1); table.put(put1); admin.flush(tableName); - Thread.sleep(1000); - + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 1)) { + Thread.sleep(10); + } + } Put put2 = new Put(rowd); byte[] value2 = Bytes.toBytes("1000dfsdf"); put2.add(fam, qual, HConstants.LATEST_TIMESTAMP, value2); @@ -311,8 +342,13 @@ public void testFlushAndCompactionwithCombinations() throws Exception { put.setAttribute("visibility", Bytes.toBytes("ram")); table.put(put2); admin.flush(tableName); - Thread.sleep(1000); - + regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + for (HRegion region : regions) { + Store store = region.getStore(fam); + while (!(store.getStorefilesCount() > 2)) { + Thread.sleep(10); + } + } TestCoprocessorForTags.checkTagPresence = true; Scan s = new Scan(row); s.setCaching(1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java index b929cfe7dde8..b2a4ee63a11c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Test; import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, SmallTests.class}) @@ -86,7 +85,6 @@ private int addWideContent(HRegion region) throws IOException { return count; } - @Test public void testWideScanBatching() throws IOException { final int batch = 256; try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java index fe1f740a7aae..3017323cd8fb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java @@ -95,14 +95,12 @@ public class TestHLog { private static final Log LOG = LogFactory.getLog(TestHLog.class); { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) - // .getLogger().setLevel(Level.ALL); - //((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")) + .getLogger().setLevel(Level.ALL); + ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL); } private static Configuration conf; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java index fb340932f01e..c55a50484166 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java @@ -95,11 +95,9 @@ @Category({RegionServerTests.class, LargeTests.class}) public class TestHLogSplit { { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); } private final static Log LOG = LogFactory.getLog(TestHLogSplit.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java index de46f25a87a6..9fe2e0bcc2cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -99,8 +99,6 @@ public static void setUpBeforeClass() throws Exception { conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setInt("replication.stats.thread.period.seconds", 5); conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); - conf1.setLong("replication.sleep.before.failover", 2000); - conf1.setInt("replication.source.maxretriesmultiplier", 10); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); @@ -141,9 +139,7 @@ public static void setUpBeforeClass() throws Exception { Admin admin1 = new HBaseAdmin(conf1); Admin admin2 = new HBaseAdmin(conf2); admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - utility1.waitUntilAllRegionsAssigned(tableName); admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - utility2.waitUntilAllRegionsAssigned(tableName); htable1 = new HTable(conf1, tableName); htable1.setWriteBufferSize(1024); htable2 = new HTable(conf2, tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index 43b37d01f088..084f8087d60d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -129,8 +129,6 @@ public static void setUpBeforeClass() throws Exception { ReplicationSourceDummy.class.getCanonicalName()); conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT); - conf.setLong("replication.sleep.before.failover", 2000); - conf.setInt("replication.source.maxretriesmultiplier", 10); utility = new HBaseTestingUtility(conf); utility.startMiniZKCluster(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index b9ecc1299380..7e9eaa72cb2f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -195,7 +195,7 @@ public static void setupBeforeClass() throws Exception { Coprocessor.PRIORITY_HIGHEST, 1, conf); // Wait for the ACL table to become available - TEST_UTIL.waitUntilAllRegionsAssigned(AccessControlLists.ACL_TABLE_NAME); + TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); // create a set of test users SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); @@ -1050,7 +1050,6 @@ public void testPostGrantRevoke() throws Exception { htd.addFamily(new HColumnDescriptor(family1)); htd.addFamily(new HColumnDescriptor(family2)); admin.createTable(htd); - TEST_UTIL.waitUntilAllRegionsAssigned(tableName); // create temp users User tblUser = User @@ -1325,7 +1324,6 @@ public void testPostGrantRevokeAtQualifierLevel() throws Exception { htd.addFamily(new HColumnDescriptor(family1)); htd.addFamily(new HColumnDescriptor(family2)); admin.createTable(htd); - TEST_UTIL.waitUntilAllRegionsAssigned(tableName); // create temp users User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); @@ -1441,7 +1439,6 @@ public void testPermissionList() throws Exception { htd.addFamily(new HColumnDescriptor(family2)); htd.setOwner(USER_OWNER); admin.createTable(htd); - TEST_UTIL.waitUntilAllRegionsAssigned(tableName); List perms; @@ -1977,7 +1974,6 @@ public void testGlobalAuthorizationForNewRegisteredRS() throws Exception { HTableDescriptor htd = new HTableDescriptor(TEST_TABLE2); htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); admin.createTable(htd); - TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE2); // Starting a new RegionServer. JVMClusterUtil.RegionServerThread newRsThread = hbaseCluster diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestScanEarlyTermination.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestScanEarlyTermination.java index 7b53a37284a9..f4b388280f33 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestScanEarlyTermination.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestScanEarlyTermination.java @@ -128,7 +128,8 @@ public void setUp() throws Exception { htd.setConfiguration(AccessControlConstants.CF_ATTRIBUTE_EARLY_OUT, "true"); admin.createTable(htd); - TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE.getTableName()); + + TEST_UTIL.waitTableEnabled(TEST_TABLE.getTableName()); } @After diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index a3d1aac71477..f0070d8dbc4c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -554,6 +554,7 @@ public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOExce private SnapshotBuilder createSnapshot(final String snapshotName, final int version) throws IOException { HTableDescriptor htd = createHtd(snapshotName); + htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); RegionData[] regions = createTable(htd, TEST_NUM_REGIONS); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 4bcd1018a3ec..bbcf7b96e323 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -85,11 +85,9 @@ public class TestFlushSnapshotFromClient { */ @BeforeClass public static void setupCluster() throws Exception { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL); + ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); setupConf(UTIL.getConfiguration()); UTIL.startMiniCluster(NUM_RS); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java index a99daf2d9e8c..e0e8bdb37451 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Comparator; -import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -38,7 +37,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -252,105 +250,6 @@ public HTableDescriptor get(TableName tablename) htds.cachehits >= (count * 2)); } - @Test - public void testHTableDescriptorsNoCache() - throws IOException, InterruptedException { - final String name = "testHTableDescriptorsNoCache"; - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - // Cleanup old tests if any debris laying around. - Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir, - false, false); - final int count = 10; - // Write out table infos. - for (int i = 0; i < count; i++) { - HTableDescriptor htd = new HTableDescriptor(name + i); - htds.createTableDescriptor(htd); - } - - for (int i = 0; i < 2 * count; i++) { - assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i % 2))); - } - // Update the table infos - for (int i = 0; i < count; i++) { - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i)); - htd.addFamily(new HColumnDescriptor("" + i)); - htds.updateTableDescriptor(new TableDescriptor(htd)); - } - for (int i = 0; i < count; i++) { - assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i))); - assertTrue("Column Family " + i + " missing", - htds.get(TableName.valueOf(name + i)).hasFamily(Bytes.toBytes("" + i))); - } - assertEquals(count * 4, htds.invocations); - assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits); - } - - @Test - public void testGetAll() - throws IOException, InterruptedException { - final String name = "testGetAll"; - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - // Cleanup old tests if any debris laying around. - Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir); - final int count = 4; - // Write out table infos. - for (int i = 0; i < count; i++) { - HTableDescriptor htd = new HTableDescriptor(name + i); - htds.createTableDescriptor(htd); - } - // add hbase:meta - HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName()); - htds.createTableDescriptor(htd); - - assertEquals("getAll() didn't return all TableDescriptors, expected: " + - (count + 1) + " got: " + htds.getAll().size(), - count + 1, htds.getAll().size()); - - } - - @Test - public void testCacheConsistency() - throws IOException, InterruptedException { - final String name = "testCacheConsistency"; - FileSystem fs = FileSystem.get(UTIL.getConfiguration()); - // Cleanup old tests if any debris laying around. - Path rootdir = new Path(UTIL.getDataTestDir(), name); - FSTableDescriptors chtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir); - FSTableDescriptors nonchtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, - rootdir, false, false); - - final int count = 10; - // Write out table infos via non-cached FSTableDescriptors - for (int i = 0; i < count; i++) { - HTableDescriptor htd = new HTableDescriptor(name + i); - nonchtds.createTableDescriptor(htd); - } - - // Calls to getAll() won't increase the cache counter, do per table. - for (int i = 0; i < count; i++) { - assertTrue(chtds.get(TableName.valueOf(name + i)) != null); - } - - assertTrue(nonchtds.getAll().size() == chtds.getAll().size()); - - // add a new entry for hbase:meta - HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName()); - nonchtds.createTableDescriptor(htd); - - // hbase:meta will only increase the cachehit by 1 - assertTrue(nonchtds.getAll().size() == chtds.getAll().size()); - - for (Map.Entry entry: nonchtds.getAll().entrySet()) { - String t = (String) entry.getKey(); - HTableDescriptor nchtd = (HTableDescriptor) entry.getValue(); - assertTrue("expected " + htd.toString() + - " got: " + chtds.get(TableName.valueOf(t)).toString(), - (nchtd.equals(chtds.get(TableName.valueOf(t))))); - } - } - @Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; @@ -435,25 +334,5 @@ public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException assertEquals(td, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); } - private static class FSTableDescriptorsTest extends FSTableDescriptors { - - public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir) - throws IOException { - this(conf, fs, rootdir, false, true); - } - - public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir, - boolean fsreadonly, boolean usecache) throws IOException { - super(conf, fs, rootdir, fsreadonly, usecache); - } - - @Override - public HTableDescriptor get(TableName tablename) - throws TableExistsException, FileNotFoundException, IOException { - LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") + - " HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits); - return super.get(tablename); - } - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 2f7051ebe1b2..d3e62fe761c5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -110,7 +110,6 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -554,7 +553,6 @@ public void testParallelHbck() throws Exception { class RunHbck implements Callable{ boolean fail = true; - @Override public HBaseFsck call(){ try{ return doFsck(conf, false); @@ -2555,20 +2553,4 @@ public void testRegionBoundariesCheck() throws Exception { @org.junit.Rule public TestName name = new TestName(); - - @Test - public void testReadOnlyProperty() throws Exception { - HBaseFsck hbck = doFsck(conf, false); - Assert.assertEquals("shouldIgnorePreCheckPermission", true, - hbck.shouldIgnorePreCheckPermission()); - - hbck = doFsck(conf, true); - Assert.assertEquals("shouldIgnorePreCheckPermission", false, - hbck.shouldIgnorePreCheckPermission()); - - hbck = doFsck(conf, true); - hbck.setIgnorePreCheckPermission(true); - Assert.assertEquals("shouldIgnorePreCheckPermission", true, - hbck.shouldIgnorePreCheckPermission()); - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java index 2da631f5bbb2..10d7f0c63520 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMergeTool.java @@ -46,9 +46,6 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.util.ToolRunner; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; /** Test stand alone merge tool that can merge arbitrary regions */ @@ -66,7 +63,7 @@ public class TestMergeTool extends HBaseTestCase { private byte [][][] rows; private MiniDFSCluster dfsCluster = null; - @Before + @Override public void setUp() throws Exception { // Set the timeout down else this test will take a while to complete. this.conf.setLong("hbase.zookeeper.recoverable.waittime", 10); @@ -150,14 +147,15 @@ public void setUp() throws Exception { try { // Create meta region createMetaRegion(); - new FSTableDescriptors(this.conf, this.fs, testDir).createTableDescriptor( + new FSTableDescriptors(this.conf, this.fs, this.testDir).createTableDescriptor( new TableDescriptor(this.desc)); /* * Create the regions we will merge */ for (int i = 0; i < sourceRegions.length; i++) { regions[i] = - HRegion.createHRegion(this.sourceRegions[i], testDir, this.conf, this.desc); + HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf, + this.desc); /* * Insert data */ @@ -178,7 +176,7 @@ public void setUp() throws Exception { } } - @After + @Override public void tearDown() throws Exception { super.tearDown(); for (int i = 0; i < sourceRegions.length; i++) { @@ -258,7 +256,6 @@ private void verifyMerge(final HRegion merged, final int upperbound) * Test merge tool. * @throws Exception */ - @Test public void testMergeTool() throws Exception { // First verify we can read the rows from the source regions and that they // contain the right data. diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 75722137e7e0..79ebdca6cd6a 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -50,32 +50,13 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - + + org.apache.maven.plugins + maven-site-plugin + + true + + diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 52d9df7a3156..77c275986874 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -22,8 +22,8 @@ java_import org.apache.hadoop.hbase.util.Pair java_import org.apache.hadoop.hbase.util.RegionSplitter java_import org.apache.hadoop.hbase.util.Bytes -java_import org.apache.hadoop.hbase.ServerName java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription +java_import org.apache.commons.collections.MapUtils # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -212,33 +212,26 @@ def create(table_name, *args) unless arg.kind_of?(String) || arg.kind_of?(Hash) raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") end - + # First, handle all the cases where arg is a column family. if arg.kind_of?(String) or arg.has_key?(NAME) # If the arg is a string, default action is to add a column to the table. # If arg has a name, it must also be a column descriptor. - descriptor = hcd(arg, htd); - # Warn if duplicate columns are added - if htd.hasFamily(descriptor.getName) - puts "Family '" + descriptor.getNameAsString() + "' already exists, the old one will be replaced" - htd.modifyFamily(descriptor) - else - htd.addFamily(descriptor) - end + htd.addFamily(hcd(arg, htd)) has_columns = true next end if arg.has_key?(REGION_REPLICATION) region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION)) htd.setRegionReplication(region_replication) - end - + end + # Get rid of the "METHOD", which is deprecated for create. # We'll do whatever it used to do below if it's table_att. if (method = arg.delete(METHOD)) raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' end - + # The hash is not a column family. Figure out what's in it. # First, handle splits. if arg.has_key?(SPLITS_FILE) @@ -253,7 +246,7 @@ def create(table_name, *args) htd.setValue(SPLITS_FILE, arg[SPLITS_FILE]) end - if arg.has_key?(SPLITS) + if arg.has_key?(SPLITS) splits = Java::byte[][arg[SPLITS].size].new idx = 0 arg.delete(SPLITS).each do |split| @@ -269,9 +262,9 @@ def create(table_name, *args) split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) splits = split_algo.split(JInteger.valueOf(num_regions)) end - + # Done with splits; apply formerly-table_att parameters. - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] @@ -280,15 +273,15 @@ def create(table_name, *args) htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - + arg.each_key do |ignored_key| puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ]) end end - + # Fail if no column families defined raise(ArgumentError, "Table must have at least one column family") if !has_columns - + if splits.nil? # Perform the create table call @admin.createTable(htd) @@ -297,15 +290,15 @@ def create(table_name, *args) @admin.createTable(htd, splits) end end - + #---------------------------------------------------------------------------------------------- # Closes a region. # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) + if (server == nil || !closeEncodedRegion?(region_name, server)) @admin.closeRegion(region_name, server) - end + end end #---------------------------------------------------------------------------------------------- @@ -326,7 +319,7 @@ def unassign(region_name, force) def move(encoded_region_name, server = nil) @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) end - + #---------------------------------------------------------------------------------------------- # Merge two regions def merge_region(encoded_region_a_name, encoded_region_b_name, force) @@ -339,10 +332,6 @@ def describe(table_name) @admin.getTableDescriptor(table_name.to_java_bytes).to_s end - def get_column_families(table_name) - @admin.getTableDescriptor(table_name.to_java_bytes).getColumnFamilies() - end - #---------------------------------------------------------------------------------------------- # Truncates table (deletes all records by recreating the table) def truncate(table_name, conf = @conf) @@ -441,14 +430,14 @@ def alter(table_name, wait = true, *args) # Process all args args.each do |arg| - - + + # Normalize args to support column name only alter specs arg = { NAME => arg } if arg.kind_of?(String) # Normalize args to support shortcut delete syntax arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - + # There are 3 possible options. # 1) Column family spec. Distinguished by having a NAME and no METHOD. method = arg.delete(METHOD) @@ -467,12 +456,12 @@ def alter(table_name, wait = true, *args) puts "Updating all regions with the new schema..." alter_status(table_name) end - + # We bypass descriptor when adding column families; refresh it to apply other args correctly. htd = @admin.getTableDescriptor(table_name.to_java_bytes) next end - + # 2) Method other than table_att, with some args. name = arg.delete(NAME) if method != nil and method != "table_att" @@ -492,26 +481,26 @@ def alter(table_name, wait = true, *args) else raise ArgumentError, "Unknown method: #{method}" end - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) end - + if method == "delete" # We bypass descriptor when deleting column families; refresh it to apply other args correctly. htd = @admin.getTableDescriptor(table_name.to_java_bytes) end - next + next end - + # 3) Some args for the table, optionally with METHOD => table_att (deprecated) raise(ArgumentError, "NAME argument in an unexpected place") if name - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] @@ -552,17 +541,17 @@ def alter(table_name, wait = true, *args) valid_coproc_keys << key end end - + valid_coproc_keys.each do |key| arg.delete(key) end @admin.modifyTable(table_name.to_java_bytes, htd) - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) @@ -629,7 +618,7 @@ def status(format) def exists?(table_name) @admin.tableExists(table_name) end - + #---------------------------------------------------------------------------------------------- # Is table enabled def enabled?(table_name) @@ -640,7 +629,7 @@ def enabled?(table_name) #Is supplied region name is encoded region name def closeEncodedRegion?(region_name, server) @admin.closeRegionWithEncodedRegionName(region_name, server) - end + end #---------------------------------------------------------------------------------------------- # Return a new HColumnDescriptor made of passed args @@ -662,15 +651,15 @@ def hcd(arg, htd) family.setBlocksize(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE) family.setMaxVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS) family.setMinVersions(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS) - family.setKeepDeletedCells(org.apache.hadoop.hbase.KeepDeletedCells.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS).to_s.upcase)) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) + family.setKeepDeletedCells(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::KEEP_DELETED_CELLS) family.setCompressTags(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESS_TAGS) family.setPrefetchBlocksOnOpen(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN))) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::PREFETCH_BLOCKS_ON_OPEN) family.setValue(COMPRESSION_COMPACT, arg.delete(COMPRESSION_COMPACT)) if arg.include?(COMPRESSION_COMPACT) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) + else family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) end end @@ -699,7 +688,7 @@ def hcd(arg, htd) arg.each_key do |unknown_key| puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key]) end - + return family end @@ -734,7 +723,7 @@ def set_user_metadata(descriptor, metadata) descriptor.setValue(k, v) end end - + #---------------------------------------------------------------------------------------------- # Take a snapshot of specified table def snapshot(table, snapshot_name, *args) @@ -790,18 +779,6 @@ def set_descriptor_config(descriptor, config) end end - #---------------------------------------------------------------------------------------------- - # Updates the configuration of one regionserver. - def update_config(serverName) - @admin.updateConfiguration(ServerName.valueOf(serverName)); - end - - #---------------------------------------------------------------------------------------------- - # Updates the configuration of all the regionservers. - def update_all_config() - @admin.updateConfiguration(); - end - #---------------------------------------------------------------------------------------------- # Returns namespace's structure description def describe_namespace(namespace_name) diff --git a/hbase-shell/src/main/ruby/hbase/security.rb b/hbase-shell/src/main/ruby/hbase/security.rb index 154c5cad8a5c..1361c19707fb 100644 --- a/hbase-shell/src/main/ruby/hbase/security.rb +++ b/hbase-shell/src/main/ruby/hbase/security.rb @@ -156,13 +156,13 @@ def user_permission(table_regex=nil) count = 0 all_perms.each do |value| user_name = String.from_java_bytes(value.getUser) - if (table_regex != nil && isNamespace?(table_regex)) + if (isNamespace?(table_regex)) namespace = table_regex[1...table_regex.length] else namespace = (value.getTableName != nil) ? value.getTableName.getNamespaceAsString() : '' end table = (value.getTableName != nil) ? value.getTableName.getNameAsString() : '' - family = (value.getFamily != nil) ? + family = (value.getFamily != nil) ? org.apache.hadoop.hbase.util.Bytes::toStringBinary(value.getFamily) : '' qualifier = (value.getQualifier != nil) ? diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 1c10bc1717bd..1cfd3d6359f5 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -360,15 +360,6 @@ def help_footer ] ) -Shell.load_command_group( - 'configuration', - :full_name => 'ONLINE CONFIGURATION TOOLS', - :commands => %w[ - update_config - update_all_config - ] -) - Shell.load_command_group( 'quotas', :full_name => 'CLUSTER QUOTAS TOOLS', diff --git a/hbase-shell/src/main/ruby/shell/commands/describe.rb b/hbase-shell/src/main/ruby/shell/commands/describe.rb index 39a30c1da0e2..9e48491d8137 100644 --- a/hbase-shell/src/main/ruby/shell/commands/describe.rb +++ b/hbase-shell/src/main/ruby/shell/commands/describe.rb @@ -28,20 +28,17 @@ def help Alternatively, you can use the abbreviated 'desc' for the same thing. hbase> desc 't1' - hbase> desc 'ns1:t1' + hbase> desc 'nds1:t1' EOF end def command(table) now = Time.now - column_families = admin.get_column_families(table) + desc = admin.describe(table) - formatter.header(["Table " + table.to_s + " is " + if admin.enabled?(table) then "ENABLED" else "DISABLED" end]) - formatter.header([ "COLUMN FAMILIES DESCRIPTION" ]) - column_families.each do |column_family| - formatter.row([ column_family.to_s ], true) - end + formatter.header([ "DESCRIPTION", "ENABLED" ], [ 64 ]) + formatter.row([ desc, admin.enabled?(table).to_s ], true, [ 64 ]) formatter.footer(now) end end diff --git a/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb deleted file mode 100644 index 05295b7996a4..000000000000 --- a/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb +++ /dev/null @@ -1,39 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class UpdateAllConfig < Command - def help - return <<-EOF -Reload a subset of configuration on all servers in the cluster. See -http://hbase.apache.org/book.html?dyn_config for more details. Here is how -you would run the command in the hbase shell: - hbase> update_all_config -EOF - end - - def command() - format_simple_command do - admin.update_all_config() - end - end - end - end -end diff --git a/hbase-shell/src/main/ruby/shell/commands/update_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_config.rb deleted file mode 100644 index 9f65fddd25d2..000000000000 --- a/hbase-shell/src/main/ruby/shell/commands/update_config.rb +++ /dev/null @@ -1,40 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class UpdateConfig < Command - def help - return <<-EOF -Reload a subset of configuration on server 'servername' where servername is -host, port plus startcode. For example: host187.example.com,60020,1289493121758 -See http://hbase.apache.org/book.html?dyn_config for more details. Here is how -you would run the command in the hbase shell: - hbase> update_config 'servername' -EOF - end - - def command(serverName) - format_simple_command do - admin.update_config(serverName) - end - end - end - end -end diff --git a/hbase-shell/src/main/ruby/shell/commands/user_permission.rb b/hbase-shell/src/main/ruby/shell/commands/user_permission.rb index 57d83beb4bac..7c292613f6c9 100644 --- a/hbase-shell/src/main/ruby/shell/commands/user_permission.rb +++ b/hbase-shell/src/main/ruby/shell/commands/user_permission.rb @@ -33,7 +33,7 @@ def help EOF end - def command(table_regex=nil) + def command(table_regex=".*") #format_simple_command do #admin.user_permission(table_regex) now = Time.now diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index 758091a0f82f..0815f3ac9133 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -46,25 +46,6 @@ - - maven-compiler-plugin - - - default-compile - - javac-with-errorprone - true - - - - default-testCompile - - javac-with-errorprone - true - - - - org.apache.maven.plugins maven-site-plugin @@ -93,15 +74,6 @@ true - - - listener - org.apache.hadoop.hbase.ServerResourceCheckerJUnitListener - - - - target/test-classes/webapps - @@ -168,6 +140,21 @@ org.apache.maven.plugins maven-source-plugin + + + maven-surefire-plugin + + + + listener + org.apache.hadoop.hbase.ServerResourceCheckerJUnitListener + + + + target/test-classes/webapps + + + diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/HTablePool.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/HTablePool.java index 400f10f00132..45578c83f8db 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/HTablePool.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/HTablePool.java @@ -235,7 +235,6 @@ public HTableInterface getTable(byte[] tableName) { * the proxy table user got from pool * @deprecated */ - @Deprecated public void putTable(HTableInterface table) throws IOException { // we need to be sure nobody puts a proxy implementation in the pool // but if the client code is not updated @@ -396,7 +395,6 @@ public void batch(List actions, Object[] results) throws IOExcept * @deprecated If any exception is thrown by one of the actions, there is no way to * retrieve the partially executed results. Use {@link #batch(List, Object[])} instead. */ - @Deprecated @Override public Object[] batch(List actions) throws IOException, InterruptedException { @@ -590,7 +588,6 @@ public void batchCallback(List actions, * {@link #batchCallback(List, Object[], org.apache.hadoop.hbase.client.coprocessor.Batch.Callback)} * instead. */ - @Deprecated @Override public Object[] batchCallback(List actions, Callback callback) throws IOException, InterruptedException { diff --git a/pom.xml b/pom.xml index a025cd20e925..1b97e380a0cf 100644 --- a/pom.xml +++ b/pom.xml @@ -65,14 +65,6 @@ hbase-rest hbase-checkstyle - - - - apache.snapshots - http://repository.apache.org/snapshots/ - - scm:git:git://git.apache.org/hbase.git scm:git:https://git-wip-us.apache.org/repos/asf/hbase.git @@ -424,6 +416,16 @@ org.apache.maven.plugins maven-release-plugin + 2.4.1 + - org.apache.maven.plugins maven-surefire-plugin ${surefire.version} @@ -514,12 +497,10 @@ - org.apache.maven.plugins maven-surefire-report-plugin ${surefire.version} - org.apache.maven.plugins maven-clean-plugin @@ -534,8 +515,7 @@ org.codehaus.mojo findbugs-maven-plugin - 3.0.0 - + ${findbugs-maven-plugin.version} ${project.basedir}/../dev-support/findbugs-exclude.xml true @@ -546,7 +526,7 @@ org.codehaus.mojo build-helper-maven-plugin - 1.9.1 + 1.5 maven-antrun-plugin @@ -602,7 +582,7 @@ org.apache.maven.plugins maven-eclipse-plugin - 2.9 + 2.8 org.apache.rat apache-rat-plugin - 0.11 + 0.8 **/*.versionsBackup @@ -704,43 +684,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - - - - [${maven.min.version},) - Maven is out of date. - HBase requires at least version ${maven.min.version} of Maven to properly build from source. - You appear to be using an older version. You can use either "mvn -version" or - "mvn enforcer:display-info" to verify what version is active. - See the reference guide on building for more information: http://hbase.apache.org/book.html#build - - - - - [${java.min.version},) - Java is out of date. - HBase requirs at least version ${java.min.version} of the JDK to properly build from source. - You appear to be using an older version. You can use either "mvn -version" or - "mvn enforcer:display-info" to verify what version is active. - See the reference guide on building for more information: http://hbase.apache.org/book.html#build - - - - - - - enforce - - enforce - - - - org.codehaus.mojo @@ -883,6 +826,7 @@ org.apache.maven.plugins maven-resources-plugin + ${maven.resources.plugin.version} false @@ -930,7 +874,7 @@ org.apache.maven.plugins maven-site-plugin - 3.4 + ${maven.site.version} false @@ -982,20 +926,17 @@ ${maven.build.timestamp} 1.7 - - 3.0.3 - ${compileSource} - 2.5.1 + 2.4.0 3.0.0-SNAPSHOT 1.2 - 1.9 + 1.7 2.4 2.6 - 1.2 - 2.2 - 3.3.0 + 1.1.3 + 2.1 + 3.2.0 3.2.1 3.1 2.2.0 @@ -1010,23 +951,26 @@ 4.11 3.0.4 1.2.17 - 1.10.8 + 1.9.0 2.5.0 0.9.0 3.4.6 - 1.7.7 + 1.7.5 0.0.1-SNAPSHOT 2.6.3 2.3.1 - 1.3.3 - 4.0.23.Final + 1.3.1 + 4.0.19.Final 2.1.2 2.4 1.6 2.3.4 + 2.5.2 1.3.9-1 + 3.3 2.9 + 2.6 /usr /etc/hbase @@ -1040,8 +984,7 @@ hbase-server-${project.version}-tests.jar hbase-common-${project.version}-tests.jar hbase-it-${project.version}-tests.jar - hbase-annotations-${project.version}-tests.jar - 2.18-SNAPSHOT + 2.17 surefire-junit47 false @@ -1085,7 +1028,7 @@ hbase-annotations ${project.version} test-jar - + test org.apache.hbase diff --git a/src/main/docbkx/configuration.xml b/src/main/docbkx/configuration.xml index 74b8e526c60e..d3959b682f7b 100644 --- a/src/main/docbkx/configuration.xml +++ b/src/main/docbkx/configuration.xml @@ -368,7 +368,7 @@ hadoop - nproc 32000 Hadoop 2.x is recommended. Hadoop 2.x is faster and includes features, such as short-circuit reads, which will help improve your HBase random read profile. Hadoop 2.x also includes important bug fixes - that will improve your overall HBase experience. HBase 0.98 drops support for Hadoop 1.0, deprecates use of Hadoop 1.1+, + that will improve your overall HBase experience. HBase 0.98 deprecates use of Hadoop 1.x, and HBase 1.0 will not support Hadoop 1.x. Use the following legend to interpret this table: @@ -411,7 +411,7 @@ hadoop - nproc 32000 HBase-0.92.x HBase-0.94.x HBase-0.96.x - HBase-0.98.x (Support for Hadoop 1.1+ is deprecated.) + HBase-0.98.x (Support for Hadoop 1.x is deprecated.) HBase-1.0.x (Hadoop 1.x is NOT supported) @@ -433,19 +433,29 @@ hadoop - nproc 32000 X - Hadoop-1.0.x + Hadoop-1.0.0-1.0.2 (HBase requires hadoop 1.0.3 at a minimum; there is an + issue where we cannot find KerberosUtil compiling against earlier versions of + Hadoop.) X X X X X + + Hadoop-1.0.3+ + S + S + S + X + X + Hadoop-1.1.x NT S S - NT + X X @@ -661,14 +671,6 @@ Index: pom.xml -
- ZooKeeper Requirements - ZooKeeper 3.4.x is required as of HBase 1.0.0. HBase makes use of the - multi functionality that is only available since 3.4.0 - (The useMulti is defaulted true in HBase 1.0.0). - See HBASE-12241 The crash of regionServer when taking deadserver's replication queue breaks replication - and Use ZK.multi when available for HBASE-6710 0.92/0.94 compatibility fix for background. -
-
- Dynamic Configuration - Changing Configuration Without Restarting Servers - Since HBase 1.0.0, it is possible to change a subset of the configuration without - requiring a server restart. In the hbase shell, there are new operators, - update_config and update_all_config that - will prompt a server or all servers to reload configuration. - Only a subset of all configurations can currently be changed in the running server. - Here is an incomplete list: - hbase.regionserver.thread.compaction.large, - hbase.regionserver.thread.compaction.small, - hbase.regionserver.thread.split, - hbase.regionserver.thread.merge, as well as compaction - policy and configurations and adjustment to offpeak hours. - For the full list consult the patch attached to - HBASE-12147 Porting Online Config Change from 89-fb. - -
diff --git a/src/main/docbkx/developer.xml b/src/main/docbkx/developer.xml index 4a59cf455969..382367f5717d 100644 --- a/src/main/docbkx/developer.xml +++ b/src/main/docbkx/developer.xml @@ -1780,40 +1780,6 @@ justification="I know what I'm doing")
-
- Git Best Practices - - - Use the correct method to create patches. See . - - - Avoid git merges. Use git pull --rebase or git - fetch followed by git rebase. - - - Do not use git push --force. If the push does not work, fix - the problem or ask for help. - - - Please contribute to this document if you think of other Git best - practices. -
- <code>rebase_all_git_branches.sh</code> - The dev-support/rebase_all_git_branches.sh script is - provided to help keep your Git repository clean. Use the -h - parameter to get usage instructions. The script automatically refreshes your - tracking branches, attempts an automatic rebase of each local branch against its - remote branch, and gives you the option to delete any branch which represents a - closed HBASE- JIRA. The script has one optional configuration - option, the location of your Git directory. You can set a default by editing the - script. Otherwise, you can pass the git directory manually by using the - -d parameter, followed by an absolute or relative directory - name, or even '.' for the current working directory. The script checks the - directory for sub-directory called .git/, before - proceeding. -
-
Submitting Patches @@ -1821,8 +1787,7 @@ justification="I know what I'm doing") contribute patches in our new GIT context, caveat the fact that we have a different branching model and that we don't currently do the merge practice described in the following, the accumulo doc - on how to contribute and develop after our move to GIT is worth a read. - See also . + on how to contribute and develop after our move to GIT is worth a read. If you are new to submitting patches to open source or new to submitting patches to Apache, start by reading the Apache HBase Project.
Create Patch - The script dev-support/make_patch.sh has been provided to - help you adhere to patch-creation guidelines. The script has the following syntax: - - $ make_patch.sh [-a] [-p <patch_dir>] - - - If you do not pass a patch_dir, the script defaults to - ~/patches/. If the patch_dir does - not exist, it is created. - - - By default, if an existing patch exists with the JIRA ID, the version - of the new patch is incremented - (HBASE-XXXX-v3.patch). If the -a - option is passed, the version is not incremented, but the suffix - -addendum is added - (HBASE-XXXX-v2-addendum.patch). A second - addendum to a given version is not supported. - - - Detects whether you have more than one local commit on your branch. If - you do, the script offers you the chance to run git rebase - -i to squash the changes into a single commit so that it - can use git format-patch. If you decline, the script - uses git diff instead. The patch is saved in a - configurable directory and is ready to be attached to your JIRA. - - Patching Workflow @@ -1933,8 +1870,8 @@ justification="I know what I'm doing") Git git format-patch is preferred because it preserves - commit messages. Use git rebase -i first, to combine - (squash) smaller commits into a single larger one. + commit messages. Use git squash first, to combine + smaller commits into a single larger one. Do not use --no-prefix, even if you were in the habit of doing it previously. diff --git a/src/main/docbkx/ops_mgt.xml b/src/main/docbkx/ops_mgt.xml index 81be6c7f1dbc..cd6562f71f08 100644 --- a/src/main/docbkx/ops_mgt.xml +++ b/src/main/docbkx/ops_mgt.xml @@ -1122,164 +1122,17 @@ $ for i in `cat conf/regionservers|sort`; do ./bin/graceful_stop.sh --restart --
-
- Most Important Master Metrics - Note: Counts are usually over the last metrics reporting interval. - - - hbase.master.numRegionServers - Number of live regionservers - - - hbase.master.numDeadRegionServers - Number of dead regionservers - - - hbase.master.ritCount - The number of regions in transition - - - hbase.master.ritCountOverThreshold - The number of regions that have been in transition longer than - a threshold time (default: 60 seconds) - - - hbase.master.ritOldestAge - The age of the longest region in transition, in milliseconds - - - -
Most Important RegionServer Metrics - Note: Counts are usually over the last metrics reporting interval. - - - hbase.regionserver.regionCount - The number of regions hosted by the regionserver - - - hbase.regionserver.storeFileCount - The number of store files on disk currently managed by the - regionserver - - - hbase.regionserver.storeFileSize - Aggregate size of the store files on disk - - - hbase.regionserver.hlogFileCount - The number of write ahead logs not yet archived - - - hbase.regionserver.totalRequestCount - The total number of requests received - - - hbase.regionserver.readRequestCount - The number of read requests received - - - hbase.regionserver.writeRequestCount - The number of write requests received - - - hbase.regionserver.numOpenConnections - The number of open connections at the RPC layer - - - hbase.regionserver.numActiveHandler - The number of RPC handlers actively servicing requests - - - hbase.regionserver.numCallsInGeneralQueue - The number of currently enqueued user requests - - - hbase.regionserver.numCallsInReplicationQueue - The number of currently enqueued operations received from - replication - - - hbase.regionserver.numCallsInPriorityQueue - The number of currently enqueued priority (internal housekeeping) - requests - - - hbase.regionserver.flushQueueLength - Current depth of the memstore flush queue. If increasing, we are falling - behind with clearing memstores out to HDFS. - - - hbase.regionserver.updatesBlockedTime - Number of milliseconds updates have been blocked so the memstore can be - flushed - - - hbase.regionserver.compactionQueueLength - Current depth of the compaction request queue. If increasing, we are - falling behind with storefile compaction. - - - hbase.regionserver.blockCacheHitCount - The number of block cache hits - - - hbase.regionserver.blockCacheMissCount - The number of block cache misses - - - hbase.regionserver.blockCacheExpressHitPercent - The percent of the time that requests with the cache turned on hit the - cache - - - hbase.regionserver.percentFilesLocal - Percent of store file data that can be read from the local DataNode, - 0-100 - - - hbase.regionserver.<op>_<measure> - Operation latencies, where <op> is one of Append, Delete, Mutate, - Get, Replay, Increment; and where <measure> is one of min, max, mean, median, - 75th_percentile, 95th_percentile, 99th_percentile - - - hbase.regionserver.slow<op>Count - The number of operations we thought were slow, where <op> is one - of the list above - - - hbase.regionserver.GcTimeMillis - Time spent in garbage collection, in milliseconds - - - hbase.regionserver.GcTimeMillisParNew - Time spent in garbage collection of the young generation, in - milliseconds - - - hbase.regionserver.GcTimeMillisConcurrentMarkSweep - Time spent in garbage collection of the old generation, in - milliseconds - - - hbase.regionserver.authenticationSuccesses - Number of client connections where authentication succeeded - - - hbase.regionserver.authenticationFailures - Number of client connection authentication failures - - - hbase.regionserver.mutationsWithoutWALCount - Count of writes submitted with a flag indicating they should bypass the - write ahead log - - + Previously, this section contained a list of the most important RegionServer metrics. + However, the list was extremely out of date. In some cases, the name of a given metric has + changed. In other cases, the metric seems to no longer be exposed. An effort is underway to + create automatic documentation for each metric based upon information pulled from its + implementation.
+
HBase Monitoring diff --git a/src/main/docbkx/troubleshooting.xml b/src/main/docbkx/troubleshooting.xml index 6ca4d06ed031..c2463b0887db 100644 --- a/src/main/docbkx/troubleshooting.xml +++ b/src/main/docbkx/troubleshooting.xml @@ -607,20 +607,6 @@ java.lang.Thread.State: WAITING (on object monitor) See .
-
- Performance Differences in Thrift and Java APIs - Poor performance, or even ScannerTimeoutExceptions, can occur if - Scan.setCaching is too high, as discussed in . If the Thrift client uses the wrong caching - settings for a given workload, performance can suffer compared to the Java API. To set - caching for a given scan in the Thrift client, use the scannerGetList(scannerId, - numRows) method, where numRows is an integer representing the number - of rows to cache. In one case, it was found that reducing the cache for Thrift scans from - 1000 to 100 increased performance to near parity with the Java API given the same - queries. - See also Jesse Andersen's blog post - about using Scans with Thrift. -
<classname>LeaseException</classname> when calling diff --git a/src/main/docbkx/upgrading.xml b/src/main/docbkx/upgrading.xml index 82fdd1ec733a..b0a248b6eb76 100644 --- a/src/main/docbkx/upgrading.xml +++ b/src/main/docbkx/upgrading.xml @@ -76,64 +76,24 @@ compatibility between versions also mean binary compatibility?</link> discussion on the hbaes dev mailing list. </para> </section> - <section xml:id="hbase.rolling.upgrade"> - <title><firstterm>Rolling Upgrades</firstterm> - A rolling upgrade is the process by which you update the servers - in your cluster a server at a time. You can rolling upgrade across HBase versions - if they are binary or wire compatible. - See for more on what this means. - Coarsely, a rolling upgrade is a graceful stop each server, - update the software, and then restart. You do this for each server in the cluster. - Usually you upgrade the Master first and then the regionservers. - See for tools that can help use the rolling upgrade process. - - For example, in the below, hbase was symlinked to the actual hbase install. - On upgrade, before running a rolling restart over the cluser, we changed the symlink - to point at the new HBase software version and then ran - $ HADOOP_HOME=~/hadoop-2.6.0-CRC-SNAPSHOT ~/hbase/bin/rolling-restart.sh --config ~/conf_hbase - The rolling-restart script will first gracefully stop and restart the master, and then - each of the regionservers in turn. Because the symlink was changed, on restart the - server will come up using the new hbase version. Check logs for errors as the - rolling upgrade proceeds. -
- Rolling Upgrade between versions that are Binary/Wire compatibile - Unless otherwise specified, HBase point versions are binary compatible. You can do - a between hbase point versions. - For example, you can go to 0.94.6 from 0.94.5 by doing a rolling upgrade - across the cluster replacing the 0.94.5 binary with a 0.94.6 binary. - In the minor version-particular sections below, we call out where the versions - are wire/protocol compatible and in this case, it is also possible to do a - . For example, in - , we - state that it is possible to do a rolling upgrade between hbase-0.98.x and hbase-1.0.0. -
+ Rolling Upgrade between versions/Binary compatibility + Unless otherwise specified, HBase point versions are binary compatible. you can do + a rolling upgrade between hbase point versions; for example, you can go to 0.94.6 + from 0.94.5 by doing a rolling upgrade across the cluster replacing the 0.94.5 + binary with a 0.94.6 binary.
-
+
Upgrading from 0.98.x to 1.0.x - In this section we first note the significant changes that come in with 1.0.0 HBase and then - we go over the upgrade process. Be sure to read the significant changes section with care - so you avoid surprises. - + A rolling upgrade from 0.98.x to 1.0.x works (TODO: Verify!!!!). The two versions are not binary + compatible.
Changes of Note! In here we list important changes that are in 1.0.0 since 0.98.x., changes you should be aware that will go into effect once you upgrade. -
ZooKeeper 3.4 is required in HBase 1.0.0 - See . -
-
HBase Default Ports Changed - The ports used by HBase changed. The used to be in the 600XX range. In - hbase-1.0.0 they have been moved up out of the ephemeral port range and are - 160XX instead (Master web UI was 60010 and is now 16030; the RegionServer - web UI was 60030 and is now 16030, etc). If you want to keep the old port - locations, copy the port setting configs from hbase-default.xml - into hbase-site.xml, change them back to the old values - from hbase-0.98.x era, and ensure you've distributed your configurations before - you restart. -
hbase.bucketcache.percentage.in.combinedcache configuration has been REMOVED You may have made use of this configuration if you are using BucketCache. @@ -153,37 +113,6 @@ HBASE-11520 Simplify offheap cache config by removing the confusing "hbase.bucketcache.percentage.in.combinedcache".
-
If you have your own customer filters.... - See the release notes on the issue HBASE-12068 [Branch-1] Avoid need to always do KeyValueUtil#ensureKeyValue for Filter transformCell; - be sure to follow the recommendations therein. - -
-
-
- Rolling upgrade from 0.98.x to HBase 1.0.0 - From 0.96.x to 1.0.0 - You cannot do a from 0.96.x to 1.0.0 without - first doing a rolling upgrade to 0.98.x. See comment in - HBASE-11164 Document and test rolling updates from 0.98 -> 1.0 for the why. - Also because hbase-1.0.0 enables hfilev3 by default, - HBASE-9801 Change the default HFile version to V3, - and support for hfilev3 only arrives in 0.98, this is another reason you cannot rolling upgrade from hbase-0.96.x; - if the rolling upgrade stalls, the 0.96.x servers cannot open files written by the servers running the newer hbase-1.0.0 - hfilev3 writing servers. - - - There are no known issues running a from hbase-0.98.x to hbase-1.0.0. - - -
-
- Upgrading to 1.0 from 0.94 - You cannot rolling upgrade from 0.94.x to 1.x.x. You must stop your cluster, - install the 1.x.x software, run the migration described at - (substituting 1.x.x. wherever we make mention of 0.96.x in the section below), - and then restart. Be sure to upgrade your zookeeper if it is a version less than the required 3.4.x. - -
@@ -213,12 +142,11 @@
Upgrading from 0.94.x to 0.96.x - The "Singularity" + The Singularity HBase 0.96.x was EOL'd, September 1st, 2014 Do not deploy 0.96.x Deploy a 0.98.x at least. See EOL 0.96. - You will have to stop your old 0.94.x cluster completely to upgrade. If you are replicating between clusters, both clusters will have to go down to upgrade. Make sure it is a clean shutdown. The less WAL files around, the faster the upgrade will run (the @@ -226,10 +154,9 @@ process). All clients must be upgraded to 0.96 too. The API has changed. You will need to recompile your code against 0.96 and you may need to adjust applications to go against new APIs (TODO: List of changes). -
+
Executing the 0.96 Upgrade - HDFS and ZooKeeper must be up! HDFS and ZooKeeper should be up and running during the upgrade process. hbase-0.96.0 comes with an upgrade script. Run