001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hbase; 019 020import java.io.IOException; 021import java.lang.reflect.InvocationTargetException; 022import java.lang.reflect.Method; 023import java.util.Map; 024 025import org.apache.hadoop.conf.Configuration; 026import org.apache.hadoop.hbase.util.VersionInfo; 027import org.apache.hadoop.hbase.zookeeper.ZKConfig; 028import org.apache.yetus.audience.InterfaceAudience; 029import org.slf4j.Logger; 030import org.slf4j.LoggerFactory; 031 032/** 033 * Adds HBase configuration files to a Configuration 034 */ 035@InterfaceAudience.Public 036public class HBaseConfiguration extends Configuration { 037 private static final Logger LOG = LoggerFactory.getLogger(HBaseConfiguration.class); 038 039 /** 040 * Instantiating HBaseConfiguration() is deprecated. Please use 041 * HBaseConfiguration#create() to construct a plain Configuration 042 * @deprecated Please use create() instead. 043 */ 044 @Deprecated 045 public HBaseConfiguration() { 046 //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration 047 super(); 048 addHbaseResources(this); 049 LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use" 050 + " HBaseConfiguration#create() to construct a plain Configuration"); 051 } 052 053 /** 054 * Instantiating HBaseConfiguration() is deprecated. Please use 055 * HBaseConfiguration#create(conf) to construct a plain Configuration 056 * @deprecated Please user create(conf) instead. 057 */ 058 @Deprecated 059 public HBaseConfiguration(final Configuration c) { 060 //TODO:replace with private constructor 061 this(); 062 merge(this, c); 063 } 064 065 private static void checkDefaultsVersion(Configuration conf) { 066 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; 067 String defaultsVersion = conf.get("hbase.defaults.for.version"); 068 String thisVersion = VersionInfo.getVersion(); 069 if (!thisVersion.equals(defaultsVersion)) { 070 throw new RuntimeException( 071 "hbase-default.xml file seems to be for an older version of HBase (" + 072 defaultsVersion + "), this version is " + thisVersion); 073 } 074 } 075 076 public static Configuration addHbaseResources(Configuration conf) { 077 conf.addResource("hbase-default.xml"); 078 conf.addResource("hbase-site.xml"); 079 080 checkDefaultsVersion(conf); 081 return conf; 082 } 083 084 /** 085 * Creates a Configuration with HBase resources 086 * @return a Configuration with HBase resources 087 */ 088 public static Configuration create() { 089 Configuration conf = new Configuration(); 090 // In case HBaseConfiguration is loaded from a different classloader than 091 // Configuration, conf needs to be set with appropriate class loader to resolve 092 // HBase resources. 093 conf.setClassLoader(HBaseConfiguration.class.getClassLoader()); 094 return addHbaseResources(conf); 095 } 096 097 /** 098 * @param that Configuration to clone. 099 * @return a Configuration created with the hbase-*.xml files plus 100 * the given configuration. 101 */ 102 public static Configuration create(final Configuration that) { 103 Configuration conf = create(); 104 merge(conf, that); 105 return conf; 106 } 107 108 /** 109 * Merge two configurations. 110 * @param destConf the configuration that will be overwritten with items 111 * from the srcConf 112 * @param srcConf the source configuration 113 **/ 114 public static void merge(Configuration destConf, Configuration srcConf) { 115 for (Map.Entry<String, String> e : srcConf) { 116 destConf.set(e.getKey(), e.getValue()); 117 } 118 } 119 120 /** 121 * Returns a subset of the configuration properties, matching the given key prefix. 122 * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix", 123 * the entry "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's 124 * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be 125 * included in the results, since it would show up as an entry with an empty key. 126 */ 127 public static Configuration subset(Configuration srcConf, String prefix) { 128 Configuration newConf = new Configuration(false); 129 for (Map.Entry<String, String> entry : srcConf) { 130 if (entry.getKey().startsWith(prefix)) { 131 String newKey = entry.getKey().substring(prefix.length()); 132 // avoid entries that would produce an empty key 133 if (!newKey.isEmpty()) { 134 newConf.set(newKey, entry.getValue()); 135 } 136 } 137 } 138 return newConf; 139 } 140 141 /** 142 * Sets all the entries in the provided {@code Map<String, String>} as properties in the 143 * given {@code Configuration}. Each property will have the specified prefix prepended, 144 * so that the configuration entries are keyed by {@code prefix + entry.getKey()}. 145 */ 146 public static void setWithPrefix(Configuration conf, String prefix, 147 Iterable<Map.Entry<String, String>> properties) { 148 for (Map.Entry<String, String> entry : properties) { 149 conf.set(prefix + entry.getKey(), entry.getValue()); 150 } 151 } 152 153 /** 154 * @return whether to show HBase Configuration in servlet 155 */ 156 public static boolean isShowConfInServlet() { 157 boolean isShowConf = false; 158 try { 159 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { 160 isShowConf = true; 161 } 162 } catch (LinkageError e) { 163 // should we handle it more aggressively in addition to log the error? 164 LOG.warn("Error thrown: ", e); 165 } catch (ClassNotFoundException ce) { 166 LOG.debug("ClassNotFound: ConfServlet"); 167 // ignore 168 } 169 return isShowConf; 170 } 171 172 /** 173 * Get the value of the <code>name</code> property as an <code>int</code>, possibly 174 * referring to the deprecated name of the configuration property. 175 * If no such property exists, the provided default value is returned, 176 * or if the specified value is not a valid <code>int</code>, 177 * then an error is thrown. 178 * 179 * @param name property name. 180 * @param deprecatedName a deprecatedName for the property to use 181 * if non-deprecated name is not used 182 * @param defaultValue default value. 183 * @throws NumberFormatException when the value is invalid 184 * @return property value as an <code>int</code>, 185 * or <code>defaultValue</code>. 186 */ 187 // TODO: developer note: This duplicates the functionality of deprecated 188 // property support in Configuration in Hadoop 2. But since Hadoop-1 does not 189 // contain these changes, we will do our own as usual. Replace these when H2 is default. 190 public static int getInt(Configuration conf, String name, 191 String deprecatedName, int defaultValue) { 192 if (conf.get(deprecatedName) != null) { 193 LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\"" 194 , deprecatedName, name)); 195 return conf.getInt(deprecatedName, defaultValue); 196 } else { 197 return conf.getInt(name, defaultValue); 198 } 199 } 200 201 /** 202 * Get the password from the Configuration instance using the 203 * getPassword method if it exists. If not, then fall back to the 204 * general get method for configuration elements. 205 * 206 * @param conf configuration instance for accessing the passwords 207 * @param alias the name of the password element 208 * @param defPass the default password 209 * @return String password or default password 210 * @throws IOException 211 */ 212 public static String getPassword(Configuration conf, String alias, 213 String defPass) throws IOException { 214 String passwd = null; 215 try { 216 Method m = Configuration.class.getMethod("getPassword", String.class); 217 char[] p = (char[]) m.invoke(conf, alias); 218 if (p != null) { 219 LOG.debug(String.format("Config option \"%s\" was found through" + 220 " the Configuration getPassword method.", alias)); 221 passwd = new String(p); 222 } else { 223 LOG.debug(String.format( 224 "Config option \"%s\" was not found. Using provided default value", 225 alias)); 226 passwd = defPass; 227 } 228 } catch (NoSuchMethodException e) { 229 // this is a version of Hadoop where the credential 230 //provider API doesn't exist yet 231 LOG.debug(String.format( 232 "Credential.getPassword method is not available." + 233 " Falling back to configuration.")); 234 passwd = conf.get(alias, defPass); 235 } catch (SecurityException e) { 236 throw new IOException(e.getMessage(), e); 237 } catch (IllegalAccessException e) { 238 throw new IOException(e.getMessage(), e); 239 } catch (IllegalArgumentException e) { 240 throw new IOException(e.getMessage(), e); 241 } catch (InvocationTargetException e) { 242 throw new IOException(e.getMessage(), e); 243 } 244 return passwd; 245 } 246 247 /** 248 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key 249 * to the base Configuration. Note that additional configuration properties may be needed 250 * for a remote cluster, so it is preferable to use 251 * {@link #createClusterConf(Configuration, String, String)}. 252 * 253 * @param baseConf the base configuration to use, containing prefixed override properties 254 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 255 * 256 * @return the merged configuration with override properties and cluster key applied 257 * 258 * @see #createClusterConf(Configuration, String, String) 259 */ 260 public static Configuration createClusterConf(Configuration baseConf, String clusterKey) 261 throws IOException { 262 return createClusterConf(baseConf, clusterKey, null); 263 } 264 265 /** 266 * Generates a {@link Configuration} instance by applying property overrides prefixed by 267 * a cluster profile key to the base Configuration. Override properties are extracted by 268 * the {@link #subset(Configuration, String)} method, then the merged on top of the base 269 * Configuration and returned. 270 * 271 * @param baseConf the base configuration to use, containing prefixed override properties 272 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 273 * @param overridePrefix the property key prefix to match for override properties, 274 * or {@code null} if none 275 * @return the merged configuration with override properties and cluster key applied 276 */ 277 public static Configuration createClusterConf(Configuration baseConf, String clusterKey, 278 String overridePrefix) throws IOException { 279 Configuration clusterConf = HBaseConfiguration.create(baseConf); 280 if (clusterKey != null && !clusterKey.isEmpty()) { 281 applyClusterKeyToConf(clusterConf, clusterKey); 282 } 283 284 if (overridePrefix != null && !overridePrefix.isEmpty()) { 285 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix); 286 HBaseConfiguration.merge(clusterConf, clusterSubset); 287 } 288 return clusterConf; 289 } 290 291 /** 292 * Apply the settings in the given key to the given configuration, this is 293 * used to communicate with distant clusters 294 * @param conf configuration object to configure 295 * @param key string that contains the 3 required configuratins 296 * @throws IOException 297 */ 298 private static void applyClusterKeyToConf(Configuration conf, String key) 299 throws IOException{ 300 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); 301 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString()); 302 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort()); 303 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent()); 304 } 305 306 /** 307 * For debugging. Dump configurations to system output as xml format. 308 * Master and RS configurations can also be dumped using 309 * http services. e.g. "curl http://master:16010/dump" 310 */ 311 public static void main(String[] args) throws Exception { 312 HBaseConfiguration.create().writeXml(System.out); 313 } 314}