001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.lib.service.hadoop; 020 021import org.apache.hadoop.classification.InterfaceAudience; 022import org.apache.hadoop.conf.Configuration; 023import org.apache.hadoop.fs.CommonConfigurationKeysPublic; 024import org.apache.hadoop.fs.FileSystem; 025import org.apache.hadoop.fs.Path; 026import org.apache.hadoop.fs.permission.FsPermission; 027import org.apache.hadoop.lib.server.BaseService; 028import org.apache.hadoop.lib.server.ServiceException; 029import org.apache.hadoop.lib.service.FileSystemAccess; 030import org.apache.hadoop.lib.service.FileSystemAccessException; 031import org.apache.hadoop.lib.service.Instrumentation; 032import org.apache.hadoop.lib.service.Scheduler; 033import org.apache.hadoop.lib.util.Check; 034import org.apache.hadoop.lib.util.ConfigurationUtils; 035import org.apache.hadoop.security.UserGroupInformation; 036import org.apache.hadoop.util.StringUtils; 037import org.apache.hadoop.util.VersionInfo; 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041import java.io.File; 042import java.io.IOException; 043import java.net.URI; 044import java.security.PrivilegedExceptionAction; 045import java.util.Collection; 046import java.util.HashSet; 047import java.util.Map; 048import java.util.Set; 049import java.util.concurrent.ConcurrentHashMap; 050import java.util.concurrent.TimeUnit; 051import java.util.concurrent.atomic.AtomicInteger; 052 053import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; 054 055@InterfaceAudience.Private 056public class FileSystemAccessService extends BaseService implements FileSystemAccess { 057 private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class); 058 059 public static final String PREFIX = "hadoop"; 060 061 private static final String INSTRUMENTATION_GROUP = "hadoop"; 062 063 public static final String AUTHENTICATION_TYPE = "authentication.type"; 064 public static final String KERBEROS_KEYTAB = "authentication.kerberos.keytab"; 065 public static final String KERBEROS_PRINCIPAL = "authentication.kerberos.principal"; 066 public static final String FS_CACHE_PURGE_FREQUENCY = "filesystem.cache.purge.frequency"; 067 public static final String FS_CACHE_PURGE_TIMEOUT = "filesystem.cache.purge.timeout"; 068 069 public static final String NAME_NODE_WHITELIST = "name.node.whitelist"; 070 071 public static final String HADOOP_CONF_DIR = "config.dir"; 072 073 private static final String[] HADOOP_CONF_FILES = {"core-site.xml", "hdfs-site.xml"}; 074 075 private static final String FILE_SYSTEM_SERVICE_CREATED = "FileSystemAccessService.created"; 076 077 private static class CachedFileSystem { 078 private FileSystem fs; 079 private long lastUse; 080 private long timeout; 081 private int count; 082 083 public CachedFileSystem(long timeout) { 084 this.timeout = timeout; 085 lastUse = -1; 086 count = 0; 087 } 088 089 synchronized FileSystem getFileSytem(Configuration conf) 090 throws IOException { 091 if (fs == null) { 092 fs = FileSystem.get(conf); 093 } 094 lastUse = -1; 095 count++; 096 return fs; 097 } 098 099 synchronized void release() throws IOException { 100 count--; 101 if (count == 0) { 102 if (timeout == 0) { 103 fs.close(); 104 fs = null; 105 lastUse = -1; 106 } 107 else { 108 lastUse = System.currentTimeMillis(); 109 } 110 } 111 } 112 113 // to avoid race conditions in the map cache adding removing entries 114 // an entry in the cache remains forever, it just closes/opens filesystems 115 // based on their utilization. Worse case scenario, the penalty we'll 116 // pay is that the amount of entries in the cache will be the total 117 // number of users in HDFS (which seems a resonable overhead). 118 synchronized boolean purgeIfIdle() throws IOException { 119 boolean ret = false; 120 if (count == 0 && lastUse != -1 && 121 (System.currentTimeMillis() - lastUse) > timeout) { 122 fs.close(); 123 fs = null; 124 lastUse = -1; 125 ret = true; 126 } 127 return ret; 128 } 129 130 } 131 132 public FileSystemAccessService() { 133 super(PREFIX); 134 } 135 136 private Collection<String> nameNodeWhitelist; 137 138 Configuration serviceHadoopConf; 139 140 private AtomicInteger unmanagedFileSystems = new AtomicInteger(); 141 142 private ConcurrentHashMap<String, CachedFileSystem> fsCache = 143 new ConcurrentHashMap<String, CachedFileSystem>(); 144 145 private long purgeTimeout; 146 147 @Override 148 protected void init() throws ServiceException { 149 LOG.info("Using FileSystemAccess JARs version [{}]", VersionInfo.getVersion()); 150 String security = getServiceConfig().get(AUTHENTICATION_TYPE, "simple").trim(); 151 if (security.equals("kerberos")) { 152 String defaultName = getServer().getName(); 153 String keytab = System.getProperty("user.home") + "/" + defaultName + ".keytab"; 154 keytab = getServiceConfig().get(KERBEROS_KEYTAB, keytab).trim(); 155 if (keytab.length() == 0) { 156 throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_KEYTAB); 157 } 158 String principal = defaultName + "/localhost@LOCALHOST"; 159 principal = getServiceConfig().get(KERBEROS_PRINCIPAL, principal).trim(); 160 if (principal.length() == 0) { 161 throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL); 162 } 163 Configuration conf = new Configuration(); 164 conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); 165 UserGroupInformation.setConfiguration(conf); 166 try { 167 UserGroupInformation.loginUserFromKeytab(principal, keytab); 168 } catch (IOException ex) { 169 throw new ServiceException(FileSystemAccessException.ERROR.H02, ex.getMessage(), ex); 170 } 171 LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab); 172 } else if (security.equals("simple")) { 173 Configuration conf = new Configuration(); 174 conf.set(HADOOP_SECURITY_AUTHENTICATION, "simple"); 175 UserGroupInformation.setConfiguration(conf); 176 LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name")); 177 } else { 178 throw new ServiceException(FileSystemAccessException.ERROR.H09, security); 179 } 180 181 String hadoopConfDirProp = getServiceConfig().get(HADOOP_CONF_DIR, getServer().getConfigDir()); 182 File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile(); 183 if (!hadoopConfDir.exists()) { 184 hadoopConfDir = new File(getServer().getConfigDir()).getAbsoluteFile(); 185 } 186 if (!hadoopConfDir.exists()) { 187 throw new ServiceException(FileSystemAccessException.ERROR.H10, hadoopConfDir); 188 } 189 try { 190 serviceHadoopConf = loadHadoopConf(hadoopConfDir); 191 } catch (IOException ex) { 192 throw new ServiceException(FileSystemAccessException.ERROR.H11, ex.toString(), ex); 193 } 194 195 LOG.debug("FileSystemAccess FileSystem configuration:"); 196 for (Map.Entry entry : serviceHadoopConf) { 197 LOG.debug(" {} = {}", entry.getKey(), entry.getValue()); 198 } 199 setRequiredServiceHadoopConf(serviceHadoopConf); 200 201 nameNodeWhitelist = toLowerCase(getServiceConfig().getTrimmedStringCollection(NAME_NODE_WHITELIST)); 202 } 203 204 private Configuration loadHadoopConf(File dir) throws IOException { 205 Configuration hadoopConf = new Configuration(false); 206 for (String file : HADOOP_CONF_FILES) { 207 File f = new File(dir, file); 208 if (f.exists()) { 209 hadoopConf.addResource(new Path(f.getAbsolutePath())); 210 } 211 } 212 return hadoopConf; 213 } 214 215 @Override 216 public void postInit() throws ServiceException { 217 super.postInit(); 218 Instrumentation instrumentation = getServer().get(Instrumentation.class); 219 instrumentation.addVariable(INSTRUMENTATION_GROUP, "unmanaged.fs", new Instrumentation.Variable<Integer>() { 220 @Override 221 public Integer getValue() { 222 return unmanagedFileSystems.get(); 223 } 224 }); 225 instrumentation.addSampler(INSTRUMENTATION_GROUP, "unmanaged.fs", 60, new Instrumentation.Variable<Long>() { 226 @Override 227 public Long getValue() { 228 return (long) unmanagedFileSystems.get(); 229 } 230 }); 231 Scheduler scheduler = getServer().get(Scheduler.class); 232 int purgeInterval = getServiceConfig().getInt(FS_CACHE_PURGE_FREQUENCY, 60); 233 purgeTimeout = getServiceConfig().getLong(FS_CACHE_PURGE_TIMEOUT, 60); 234 purgeTimeout = (purgeTimeout > 0) ? purgeTimeout : 0; 235 if (purgeTimeout > 0) { 236 scheduler.schedule(new FileSystemCachePurger(), 237 purgeInterval, purgeInterval, TimeUnit.SECONDS); 238 } 239 } 240 241 private class FileSystemCachePurger implements Runnable { 242 243 @Override 244 public void run() { 245 int count = 0; 246 for (CachedFileSystem cacheFs : fsCache.values()) { 247 try { 248 count += cacheFs.purgeIfIdle() ? 1 : 0; 249 } catch (Throwable ex) { 250 LOG.warn("Error while purging filesystem, " + ex.toString(), ex); 251 } 252 } 253 LOG.debug("Purged [{}} filesystem instances", count); 254 } 255 } 256 257 private Set<String> toLowerCase(Collection<String> collection) { 258 Set<String> set = new HashSet<String>(); 259 for (String value : collection) { 260 set.add(StringUtils.toLowerCase(value)); 261 } 262 return set; 263 } 264 265 @Override 266 public Class getInterface() { 267 return FileSystemAccess.class; 268 } 269 270 @Override 271 public Class[] getServiceDependencies() { 272 return new Class[]{Instrumentation.class, Scheduler.class}; 273 } 274 275 protected UserGroupInformation getUGI(String user) throws IOException { 276 return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); 277 } 278 279 protected void setRequiredServiceHadoopConf(Configuration conf) { 280 conf.set("fs.hdfs.impl.disable.cache", "true"); 281 } 282 283 private static final String HTTPFS_FS_USER = "httpfs.fs.user"; 284 285 protected FileSystem createFileSystem(Configuration namenodeConf) 286 throws IOException { 287 String user = UserGroupInformation.getCurrentUser().getShortUserName(); 288 CachedFileSystem newCachedFS = new CachedFileSystem(purgeTimeout); 289 CachedFileSystem cachedFS = fsCache.putIfAbsent(user, newCachedFS); 290 if (cachedFS == null) { 291 cachedFS = newCachedFS; 292 } 293 Configuration conf = new Configuration(namenodeConf); 294 conf.set(HTTPFS_FS_USER, user); 295 return cachedFS.getFileSytem(conf); 296 } 297 298 protected void closeFileSystem(FileSystem fs) throws IOException { 299 if (fsCache.containsKey(fs.getConf().get(HTTPFS_FS_USER))) { 300 fsCache.get(fs.getConf().get(HTTPFS_FS_USER)).release(); 301 } 302 } 303 304 protected void validateNamenode(String namenode) throws FileSystemAccessException { 305 if (nameNodeWhitelist.size() > 0 && !nameNodeWhitelist.contains("*")) { 306 if (!nameNodeWhitelist.contains( 307 StringUtils.toLowerCase(namenode))) { 308 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H05, namenode, "not in whitelist"); 309 } 310 } 311 } 312 313 protected void checkNameNodeHealth(FileSystem fileSystem) throws FileSystemAccessException { 314 } 315 316 @Override 317 public <T> T execute(String user, final Configuration conf, final FileSystemExecutor<T> executor) 318 throws FileSystemAccessException { 319 Check.notEmpty(user, "user"); 320 Check.notNull(conf, "conf"); 321 Check.notNull(executor, "executor"); 322 if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) { 323 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04); 324 } 325 if (conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) == null || 326 conf.getTrimmed(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY).length() == 0) { 327 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H06, 328 CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); 329 } 330 try { 331 validateNamenode( 332 new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)). 333 getAuthority()); 334 UserGroupInformation ugi = getUGI(user); 335 return ugi.doAs(new PrivilegedExceptionAction<T>() { 336 @Override 337 public T run() throws Exception { 338 FileSystem fs = createFileSystem(conf); 339 Instrumentation instrumentation = getServer().get(Instrumentation.class); 340 Instrumentation.Cron cron = instrumentation.createCron(); 341 try { 342 checkNameNodeHealth(fs); 343 cron.start(); 344 return executor.execute(fs); 345 } finally { 346 cron.stop(); 347 instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(), cron); 348 closeFileSystem(fs); 349 } 350 } 351 }); 352 } catch (FileSystemAccessException ex) { 353 throw ex; 354 } catch (Exception ex) { 355 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H03, ex); 356 } 357 } 358 359 public FileSystem createFileSystemInternal(String user, final Configuration conf) 360 throws IOException, FileSystemAccessException { 361 Check.notEmpty(user, "user"); 362 Check.notNull(conf, "conf"); 363 if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) { 364 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04); 365 } 366 try { 367 validateNamenode( 368 new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).getAuthority()); 369 UserGroupInformation ugi = getUGI(user); 370 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 371 @Override 372 public FileSystem run() throws Exception { 373 return createFileSystem(conf); 374 } 375 }); 376 } catch (IOException ex) { 377 throw ex; 378 } catch (FileSystemAccessException ex) { 379 throw ex; 380 } catch (Exception ex) { 381 throw new FileSystemAccessException(FileSystemAccessException.ERROR.H08, ex.getMessage(), ex); 382 } 383 } 384 385 @Override 386 public FileSystem createFileSystem(String user, final Configuration conf) throws IOException, 387 FileSystemAccessException { 388 unmanagedFileSystems.incrementAndGet(); 389 return createFileSystemInternal(user, conf); 390 } 391 392 @Override 393 public void releaseFileSystem(FileSystem fs) throws IOException { 394 unmanagedFileSystems.decrementAndGet(); 395 closeFileSystem(fs); 396 } 397 398 @Override 399 public Configuration getFileSystemConfiguration() { 400 Configuration conf = new Configuration(true); 401 ConfigurationUtils.copy(serviceHadoopConf, conf); 402 conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true); 403 404 // Force-clear server-side umask to make HttpFS match WebHDFS behavior 405 conf.set(FsPermission.UMASK_LABEL, "000"); 406 407 return conf; 408 } 409 410}