001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.lib.service.hadoop;
020
021import org.apache.hadoop.classification.InterfaceAudience;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
024import org.apache.hadoop.fs.FileSystem;
025import org.apache.hadoop.fs.Path;
026import org.apache.hadoop.fs.permission.FsPermission;
027import org.apache.hadoop.lib.server.BaseService;
028import org.apache.hadoop.lib.server.ServiceException;
029import org.apache.hadoop.lib.service.FileSystemAccess;
030import org.apache.hadoop.lib.service.FileSystemAccessException;
031import org.apache.hadoop.lib.service.Instrumentation;
032import org.apache.hadoop.lib.service.Scheduler;
033import org.apache.hadoop.lib.util.Check;
034import org.apache.hadoop.lib.util.ConfigurationUtils;
035import org.apache.hadoop.security.UserGroupInformation;
036import org.apache.hadoop.util.StringUtils;
037import org.apache.hadoop.util.VersionInfo;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041import java.io.File;
042import java.io.IOException;
043import java.net.URI;
044import java.security.PrivilegedExceptionAction;
045import java.util.Collection;
046import java.util.HashSet;
047import java.util.Map;
048import java.util.Set;
049import java.util.concurrent.ConcurrentHashMap;
050import java.util.concurrent.TimeUnit;
051import java.util.concurrent.atomic.AtomicInteger;
052
053import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
054
055@InterfaceAudience.Private
056public class FileSystemAccessService extends BaseService implements FileSystemAccess {
057  private static final Logger LOG = LoggerFactory.getLogger(FileSystemAccessService.class);
058
059  public static final String PREFIX = "hadoop";
060
061  private static final String INSTRUMENTATION_GROUP = "hadoop";
062
063  public static final String AUTHENTICATION_TYPE = "authentication.type";
064  public static final String KERBEROS_KEYTAB = "authentication.kerberos.keytab";
065  public static final String KERBEROS_PRINCIPAL = "authentication.kerberos.principal";
066  public static final String FS_CACHE_PURGE_FREQUENCY = "filesystem.cache.purge.frequency";
067  public static final String FS_CACHE_PURGE_TIMEOUT = "filesystem.cache.purge.timeout";
068
069  public static final String NAME_NODE_WHITELIST = "name.node.whitelist";
070
071  public static final String HADOOP_CONF_DIR = "config.dir";
072
073  private static final String[] HADOOP_CONF_FILES = {"core-site.xml", "hdfs-site.xml"};
074
075  private static final String FILE_SYSTEM_SERVICE_CREATED = "FileSystemAccessService.created";
076
077  private static class CachedFileSystem {
078    private FileSystem fs;
079    private long lastUse;
080    private long timeout;
081    private int count;
082
083    public CachedFileSystem(long timeout) {
084      this.timeout = timeout;
085      lastUse = -1;
086      count = 0;
087    }
088
089    synchronized FileSystem getFileSytem(Configuration conf)
090      throws IOException {
091      if (fs == null) {
092        fs = FileSystem.get(conf);
093      }
094      lastUse = -1;
095      count++;
096      return fs;
097    }
098
099    synchronized void release() throws IOException {
100      count--;
101      if (count == 0) {
102        if (timeout == 0) {
103          fs.close();
104          fs = null;
105          lastUse = -1;
106        }
107        else {
108          lastUse = System.currentTimeMillis();
109        }
110      }
111    }
112
113    // to avoid race conditions in the map cache adding removing entries
114    // an entry in the cache remains forever, it just closes/opens filesystems
115    // based on their utilization. Worse case scenario, the penalty we'll
116    // pay is that the amount of entries in the cache will be the total
117    // number of users in HDFS (which seems a resonable overhead).
118    synchronized boolean purgeIfIdle() throws IOException {
119      boolean ret = false;
120      if (count == 0 && lastUse != -1 &&
121          (System.currentTimeMillis() - lastUse) > timeout) {
122        fs.close();
123        fs = null;
124        lastUse = -1;
125        ret = true;
126      }
127      return ret;
128    }
129
130  }
131
132  public FileSystemAccessService() {
133    super(PREFIX);
134  }
135
136  private Collection<String> nameNodeWhitelist;
137
138  Configuration serviceHadoopConf;
139
140  private AtomicInteger unmanagedFileSystems = new AtomicInteger();
141
142  private ConcurrentHashMap<String, CachedFileSystem> fsCache =
143    new ConcurrentHashMap<String, CachedFileSystem>();
144
145  private long purgeTimeout;
146
147  @Override
148  protected void init() throws ServiceException {
149    LOG.info("Using FileSystemAccess JARs version [{}]", VersionInfo.getVersion());
150    String security = getServiceConfig().get(AUTHENTICATION_TYPE, "simple").trim();
151    if (security.equals("kerberos")) {
152      String defaultName = getServer().getName();
153      String keytab = System.getProperty("user.home") + "/" + defaultName + ".keytab";
154      keytab = getServiceConfig().get(KERBEROS_KEYTAB, keytab).trim();
155      if (keytab.length() == 0) {
156        throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_KEYTAB);
157      }
158      String principal = defaultName + "/localhost@LOCALHOST";
159      principal = getServiceConfig().get(KERBEROS_PRINCIPAL, principal).trim();
160      if (principal.length() == 0) {
161        throw new ServiceException(FileSystemAccessException.ERROR.H01, KERBEROS_PRINCIPAL);
162      }
163      Configuration conf = new Configuration();
164      conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
165      UserGroupInformation.setConfiguration(conf);
166      try {
167        UserGroupInformation.loginUserFromKeytab(principal, keytab);
168      } catch (IOException ex) {
169        throw new ServiceException(FileSystemAccessException.ERROR.H02, ex.getMessage(), ex);
170      }
171      LOG.info("Using FileSystemAccess Kerberos authentication, principal [{}] keytab [{}]", principal, keytab);
172    } else if (security.equals("simple")) {
173      Configuration conf = new Configuration();
174      conf.set(HADOOP_SECURITY_AUTHENTICATION, "simple");
175      UserGroupInformation.setConfiguration(conf);
176      LOG.info("Using FileSystemAccess simple/pseudo authentication, principal [{}]", System.getProperty("user.name"));
177    } else {
178      throw new ServiceException(FileSystemAccessException.ERROR.H09, security);
179    }
180
181    String hadoopConfDirProp = getServiceConfig().get(HADOOP_CONF_DIR, getServer().getConfigDir());
182    File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile();
183    if (!hadoopConfDir.exists()) {
184      hadoopConfDir = new File(getServer().getConfigDir()).getAbsoluteFile();
185    }
186    if (!hadoopConfDir.exists()) {
187      throw new ServiceException(FileSystemAccessException.ERROR.H10, hadoopConfDir);
188    }
189    try {
190      serviceHadoopConf = loadHadoopConf(hadoopConfDir);
191    } catch (IOException ex) {
192      throw new ServiceException(FileSystemAccessException.ERROR.H11, ex.toString(), ex);
193    }
194
195    LOG.debug("FileSystemAccess FileSystem configuration:");
196    for (Map.Entry entry : serviceHadoopConf) {
197      LOG.debug("  {} = {}", entry.getKey(), entry.getValue());
198    }
199    setRequiredServiceHadoopConf(serviceHadoopConf);
200
201    nameNodeWhitelist = toLowerCase(getServiceConfig().getTrimmedStringCollection(NAME_NODE_WHITELIST));
202  }
203
204  private Configuration loadHadoopConf(File dir) throws IOException {
205    Configuration hadoopConf = new Configuration(false);
206    for (String file : HADOOP_CONF_FILES) {
207      File f = new File(dir, file);
208      if (f.exists()) {
209        hadoopConf.addResource(new Path(f.getAbsolutePath()));
210      }
211    }
212    return hadoopConf;
213  }
214
215  @Override
216  public void postInit() throws ServiceException {
217    super.postInit();
218    Instrumentation instrumentation = getServer().get(Instrumentation.class);
219    instrumentation.addVariable(INSTRUMENTATION_GROUP, "unmanaged.fs", new Instrumentation.Variable<Integer>() {
220      @Override
221      public Integer getValue() {
222        return unmanagedFileSystems.get();
223      }
224    });
225    instrumentation.addSampler(INSTRUMENTATION_GROUP, "unmanaged.fs", 60, new Instrumentation.Variable<Long>() {
226      @Override
227      public Long getValue() {
228        return (long) unmanagedFileSystems.get();
229      }
230    });
231    Scheduler scheduler = getServer().get(Scheduler.class);
232    int purgeInterval = getServiceConfig().getInt(FS_CACHE_PURGE_FREQUENCY, 60);
233    purgeTimeout = getServiceConfig().getLong(FS_CACHE_PURGE_TIMEOUT, 60);
234    purgeTimeout = (purgeTimeout > 0) ? purgeTimeout : 0;
235    if (purgeTimeout > 0) {
236      scheduler.schedule(new FileSystemCachePurger(),
237                         purgeInterval, purgeInterval, TimeUnit.SECONDS);
238    }
239  }
240
241  private class FileSystemCachePurger implements Runnable {
242
243    @Override
244    public void run() {
245      int count = 0;
246      for (CachedFileSystem cacheFs : fsCache.values()) {
247        try {
248          count += cacheFs.purgeIfIdle() ? 1 : 0;
249        } catch (Throwable ex) {
250          LOG.warn("Error while purging filesystem, " + ex.toString(), ex);
251        }
252      }
253      LOG.debug("Purged [{}} filesystem instances", count);
254    }
255  }
256
257  private Set<String> toLowerCase(Collection<String> collection) {
258    Set<String> set = new HashSet<String>();
259    for (String value : collection) {
260      set.add(StringUtils.toLowerCase(value));
261    }
262    return set;
263  }
264
265  @Override
266  public Class getInterface() {
267    return FileSystemAccess.class;
268  }
269
270  @Override
271  public Class[] getServiceDependencies() {
272    return new Class[]{Instrumentation.class, Scheduler.class};
273  }
274
275  protected UserGroupInformation getUGI(String user) throws IOException {
276    return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
277  }
278
279  protected void setRequiredServiceHadoopConf(Configuration conf) {
280    conf.set("fs.hdfs.impl.disable.cache", "true");
281  }
282
283  private static final String HTTPFS_FS_USER = "httpfs.fs.user";
284
285  protected FileSystem createFileSystem(Configuration namenodeConf)
286    throws IOException {
287    String user = UserGroupInformation.getCurrentUser().getShortUserName();
288    CachedFileSystem newCachedFS = new CachedFileSystem(purgeTimeout);
289    CachedFileSystem cachedFS = fsCache.putIfAbsent(user, newCachedFS);
290    if (cachedFS == null) {
291      cachedFS = newCachedFS;
292    }
293    Configuration conf = new Configuration(namenodeConf);
294    conf.set(HTTPFS_FS_USER, user);
295    return cachedFS.getFileSytem(conf);
296  }
297
298  protected void closeFileSystem(FileSystem fs) throws IOException {
299    if (fsCache.containsKey(fs.getConf().get(HTTPFS_FS_USER))) {
300      fsCache.get(fs.getConf().get(HTTPFS_FS_USER)).release();
301    }
302  }
303
304  protected void validateNamenode(String namenode) throws FileSystemAccessException {
305    if (nameNodeWhitelist.size() > 0 && !nameNodeWhitelist.contains("*")) {
306      if (!nameNodeWhitelist.contains(
307          StringUtils.toLowerCase(namenode))) {
308        throw new FileSystemAccessException(FileSystemAccessException.ERROR.H05, namenode, "not in whitelist");
309      }
310    }
311  }
312
313  protected void checkNameNodeHealth(FileSystem fileSystem) throws FileSystemAccessException {
314  }
315
316  @Override
317  public <T> T execute(String user, final Configuration conf, final FileSystemExecutor<T> executor)
318    throws FileSystemAccessException {
319    Check.notEmpty(user, "user");
320    Check.notNull(conf, "conf");
321    Check.notNull(executor, "executor");
322    if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) {
323      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04);
324    }
325    if (conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY) == null ||
326        conf.getTrimmed(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY).length() == 0) {
327      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H06,
328                                          CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
329    }
330    try {
331      validateNamenode(
332        new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).
333          getAuthority());
334      UserGroupInformation ugi = getUGI(user);
335      return ugi.doAs(new PrivilegedExceptionAction<T>() {
336        @Override
337        public T run() throws Exception {
338          FileSystem fs = createFileSystem(conf);
339          Instrumentation instrumentation = getServer().get(Instrumentation.class);
340          Instrumentation.Cron cron = instrumentation.createCron();
341          try {
342            checkNameNodeHealth(fs);
343            cron.start();
344            return executor.execute(fs);
345          } finally {
346            cron.stop();
347            instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(), cron);
348            closeFileSystem(fs);
349          }
350        }
351      });
352    } catch (FileSystemAccessException ex) {
353      throw ex;
354    } catch (Exception ex) {
355      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H03, ex);
356    }
357  }
358
359  public FileSystem createFileSystemInternal(String user, final Configuration conf)
360    throws IOException, FileSystemAccessException {
361    Check.notEmpty(user, "user");
362    Check.notNull(conf, "conf");
363    if (!conf.getBoolean(FILE_SYSTEM_SERVICE_CREATED, false)) {
364      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H04);
365    }
366    try {
367      validateNamenode(
368        new URI(conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)).getAuthority());
369      UserGroupInformation ugi = getUGI(user);
370      return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
371        @Override
372        public FileSystem run() throws Exception {
373          return createFileSystem(conf);
374        }
375      });
376    } catch (IOException ex) {
377      throw ex;
378    } catch (FileSystemAccessException ex) {
379      throw ex;
380    } catch (Exception ex) {
381      throw new FileSystemAccessException(FileSystemAccessException.ERROR.H08, ex.getMessage(), ex);
382    }
383  }
384
385  @Override
386  public FileSystem createFileSystem(String user, final Configuration conf) throws IOException,
387    FileSystemAccessException {
388    unmanagedFileSystems.incrementAndGet();
389    return createFileSystemInternal(user, conf);
390  }
391
392  @Override
393  public void releaseFileSystem(FileSystem fs) throws IOException {
394    unmanagedFileSystems.decrementAndGet();
395    closeFileSystem(fs);
396  }
397
398  @Override
399  public Configuration getFileSystemConfiguration() {
400    Configuration conf = new Configuration(true);
401    ConfigurationUtils.copy(serviceHadoopConf, conf);
402    conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true);
403
404    // Force-clear server-side umask to make HttpFS match WebHDFS behavior
405    conf.set(FsPermission.UMASK_LABEL, "000");
406
407    return conf;
408  }
409
410}