001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.fs.http.client;
019
020import java.util.ArrayList;
021import java.util.Arrays;
022import java.util.Collection;
023import java.util.EnumSet;
024import java.util.List;
025
026import org.apache.hadoop.classification.InterfaceAudience;
027import org.apache.hadoop.conf.Configuration;
028import org.apache.hadoop.fs.ContentSummary;
029import org.apache.hadoop.fs.DelegationTokenRenewer;
030import org.apache.hadoop.fs.FSDataInputStream;
031import org.apache.hadoop.fs.FSDataOutputStream;
032import org.apache.hadoop.fs.FileChecksum;
033import org.apache.hadoop.fs.FileStatus;
034import org.apache.hadoop.fs.FileSystem;
035import org.apache.hadoop.fs.Path;
036import org.apache.hadoop.fs.PositionedReadable;
037import org.apache.hadoop.fs.Seekable;
038import org.apache.hadoop.fs.StorageType;
039import org.apache.hadoop.fs.XAttrCodec;
040import org.apache.hadoop.fs.XAttrSetFlag;
041import org.apache.hadoop.fs.permission.AclEntry;
042import org.apache.hadoop.fs.permission.AclStatus;
043import org.apache.hadoop.fs.permission.FsPermission;
044import org.apache.hadoop.hdfs.DFSConfigKeys;
045import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
046import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
047import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
048import org.apache.hadoop.lib.wsrs.EnumSetParam;
049import org.apache.hadoop.security.UserGroupInformation;
050import org.apache.hadoop.security.token.Token;
051import org.apache.hadoop.security.token.TokenIdentifier;
052import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
053import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
054import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator;
055import org.apache.hadoop.util.HttpExceptionUtils;
056import org.apache.hadoop.util.Progressable;
057import org.apache.hadoop.util.ReflectionUtils;
058import org.apache.hadoop.util.StringUtils;
059import org.json.simple.JSONArray;
060import org.json.simple.JSONObject;
061import org.json.simple.parser.JSONParser;
062import org.json.simple.parser.ParseException;
063
064import com.google.common.base.Preconditions;
065import com.google.common.collect.Lists;
066import com.google.common.collect.Maps;
067
068import java.io.BufferedInputStream;
069import java.io.BufferedOutputStream;
070import java.io.DataInput;
071import java.io.DataOutput;
072import java.io.FileNotFoundException;
073import java.io.FilterInputStream;
074import java.io.IOException;
075import java.io.InputStream;
076import java.io.OutputStream;
077import java.net.HttpURLConnection;
078import java.net.URI;
079import java.net.URISyntaxException;
080import java.net.URL;
081import java.security.PrivilegedExceptionAction;
082import java.text.MessageFormat;
083import java.util.HashMap;
084import java.util.Map;
085
086/**
087 * HttpFSServer implementation of the FileSystemAccess FileSystem.
088 * <p>
089 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
090 */
091@InterfaceAudience.Private
092public class HttpFSFileSystem extends FileSystem
093  implements DelegationTokenRenewer.Renewable {
094
095  public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME;
096
097  public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION;
098
099  public static final String SCHEME = "webhdfs";
100
101  public static final String OP_PARAM = "op";
102  public static final String DO_AS_PARAM = "doas";
103  public static final String OVERWRITE_PARAM = "overwrite";
104  public static final String REPLICATION_PARAM = "replication";
105  public static final String BLOCKSIZE_PARAM = "blocksize";
106  public static final String PERMISSION_PARAM = "permission";
107  public static final String ACLSPEC_PARAM = "aclspec";
108  public static final String DESTINATION_PARAM = "destination";
109  public static final String RECURSIVE_PARAM = "recursive";
110  public static final String SOURCES_PARAM = "sources";
111  public static final String OWNER_PARAM = "owner";
112  public static final String GROUP_PARAM = "group";
113  public static final String MODIFICATION_TIME_PARAM = "modificationtime";
114  public static final String ACCESS_TIME_PARAM = "accesstime";
115  public static final String XATTR_NAME_PARAM = "xattr.name";
116  public static final String XATTR_VALUE_PARAM = "xattr.value";
117  public static final String XATTR_SET_FLAG_PARAM = "flag";
118  public static final String XATTR_ENCODING_PARAM = "encoding";
119  public static final String NEW_LENGTH_PARAM = "newlength";
120  public static final String POLICY_NAME_PARAM = "storagepolicy";
121
122  public static final Short DEFAULT_PERMISSION = 0755;
123  public static final String ACLSPEC_DEFAULT = "";
124
125  public static final String RENAME_JSON = "boolean";
126
127  public static final String TRUNCATE_JSON = "boolean";
128
129  public static final String DELETE_JSON = "boolean";
130
131  public static final String MKDIRS_JSON = "boolean";
132
133  public static final String HOME_DIR_JSON = "Path";
134
135  public static final String SET_REPLICATION_JSON = "boolean";
136
137  public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
138
139  public static enum FILE_TYPE {
140    FILE, DIRECTORY, SYMLINK;
141
142    public static FILE_TYPE getType(FileStatus fileStatus) {
143      if (fileStatus.isFile()) {
144        return FILE;
145      }
146      if (fileStatus.isDirectory()) {
147        return DIRECTORY;
148      }
149      if (fileStatus.isSymlink()) {
150        return SYMLINK;
151      }
152      throw new IllegalArgumentException("Could not determine filetype for: " +
153                                         fileStatus.getPath());
154    }
155  }
156
157  public static final String FILE_STATUSES_JSON = "FileStatuses";
158  public static final String FILE_STATUS_JSON = "FileStatus";
159  public static final String PATH_SUFFIX_JSON = "pathSuffix";
160  public static final String TYPE_JSON = "type";
161  public static final String LENGTH_JSON = "length";
162  public static final String OWNER_JSON = "owner";
163  public static final String GROUP_JSON = "group";
164  public static final String PERMISSION_JSON = "permission";
165  public static final String ACCESS_TIME_JSON = "accessTime";
166  public static final String MODIFICATION_TIME_JSON = "modificationTime";
167  public static final String BLOCK_SIZE_JSON = "blockSize";
168  public static final String REPLICATION_JSON = "replication";
169  public static final String XATTRS_JSON = "XAttrs";
170  public static final String XATTR_NAME_JSON = "name";
171  public static final String XATTR_VALUE_JSON = "value";
172  public static final String XATTRNAMES_JSON = "XAttrNames";
173
174  public static final String FILE_CHECKSUM_JSON = "FileChecksum";
175  public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
176  public static final String CHECKSUM_BYTES_JSON = "bytes";
177  public static final String CHECKSUM_LENGTH_JSON = "length";
178
179  public static final String CONTENT_SUMMARY_JSON = "ContentSummary";
180  public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount";
181  public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount";
182  public static final String CONTENT_SUMMARY_LENGTH_JSON = "length";
183  public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota";
184  public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
185  public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
186
187  public static final String ACL_STATUS_JSON = "AclStatus";
188  public static final String ACL_STICKY_BIT_JSON = "stickyBit";
189  public static final String ACL_ENTRIES_JSON = "entries";
190  public static final String ACL_BIT_JSON = "aclBit";
191
192  public static final String ENC_BIT_JSON = "encBit";
193
194  public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies";
195  public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy";
196
197  public static final int HTTP_TEMPORARY_REDIRECT = 307;
198
199  private static final String HTTP_GET = "GET";
200  private static final String HTTP_PUT = "PUT";
201  private static final String HTTP_POST = "POST";
202  private static final String HTTP_DELETE = "DELETE";
203
204  @InterfaceAudience.Private
205  public static enum Operation {
206    OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
207    GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
208    GETFILECHECKSUM(HTTP_GET),  GETFILEBLOCKLOCATIONS(HTTP_GET),
209    INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET),
210    APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST),
211    CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
212    SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
213    MODIFYACLENTRIES(HTTP_PUT), REMOVEACLENTRIES(HTTP_PUT),
214    REMOVEDEFAULTACL(HTTP_PUT), REMOVEACL(HTTP_PUT), SETACL(HTTP_PUT),
215    DELETE(HTTP_DELETE), SETXATTR(HTTP_PUT), GETXATTRS(HTTP_GET),
216    REMOVEXATTR(HTTP_PUT), LISTXATTRS(HTTP_GET),
217    GETALLSTORAGEPOLICY(HTTP_GET), GETSTORAGEPOLICY(HTTP_GET),
218    SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST);
219
220    private String httpMethod;
221
222    Operation(String httpMethod) {
223      this.httpMethod = httpMethod;
224    }
225
226    public String getMethod() {
227      return httpMethod;
228    }
229
230  }
231
232  private DelegationTokenAuthenticatedURL authURL;
233  private DelegationTokenAuthenticatedURL.Token authToken =
234      new DelegationTokenAuthenticatedURL.Token();
235  private URI uri;
236  private Path workingDir;
237  private UserGroupInformation realUser;
238
239
240
241  /**
242   * Convenience method that creates a <code>HttpURLConnection</code> for the
243   * HttpFSServer file system operations.
244   * <p>
245   * This methods performs and injects any needed authentication credentials
246   * via the {@link #getConnection(URL, String)} method
247   *
248   * @param method the HTTP method.
249   * @param params the query string parameters.
250   * @param path the file path
251   * @param makeQualified if the path should be 'makeQualified'
252   *
253   * @return a <code>HttpURLConnection</code> for the HttpFSServer server,
254   *         authenticated and ready to use for the specified path and file system operation.
255   *
256   * @throws IOException thrown if an IO error occurs.
257   */
258  private HttpURLConnection getConnection(final String method,
259      Map<String, String> params, Path path, boolean makeQualified)
260      throws IOException {
261    return getConnection(method, params, null, path, makeQualified);
262  }
263
264  /**
265   * Convenience method that creates a <code>HttpURLConnection</code> for the
266   * HttpFSServer file system operations.
267   * <p/>
268   * This methods performs and injects any needed authentication credentials
269   * via the {@link #getConnection(URL, String)} method
270   *
271   * @param method the HTTP method.
272   * @param params the query string parameters.
273   * @param multiValuedParams multi valued parameters of the query string
274   * @param path the file path
275   * @param makeQualified if the path should be 'makeQualified'
276   *
277   * @return HttpURLConnection a <code>HttpURLConnection</code> for the
278   *         HttpFSServer server, authenticated and ready to use for the
279   *         specified path and file system operation.
280   *
281   * @throws IOException thrown if an IO error occurs.
282   */
283  private HttpURLConnection getConnection(final String method,
284      Map<String, String> params, Map<String, List<String>> multiValuedParams,
285      Path path, boolean makeQualified) throws IOException {
286    if (makeQualified) {
287      path = makeQualified(path);
288    }
289    final URL url = HttpFSUtils.createURL(path, params, multiValuedParams);
290    try {
291      return UserGroupInformation.getCurrentUser().doAs(
292          new PrivilegedExceptionAction<HttpURLConnection>() {
293            @Override
294            public HttpURLConnection run() throws Exception {
295              return getConnection(url, method);
296            }
297          }
298      );
299    } catch (Exception ex) {
300      if (ex instanceof IOException) {
301        throw (IOException) ex;
302      } else {
303        throw new IOException(ex);
304      }
305    }
306  }
307
308  /**
309   * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
310   * <p>
311   * This methods performs and injects any needed authentication credentials.
312   *
313   * @param url url to connect to.
314   * @param method the HTTP method.
315   *
316   * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for
317   *         the specified path and file system operation.
318   *
319   * @throws IOException thrown if an IO error occurs.
320   */
321  private HttpURLConnection getConnection(URL url, String method) throws IOException {
322    try {
323      HttpURLConnection conn = authURL.openConnection(url, authToken);
324      conn.setRequestMethod(method);
325      if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
326        conn.setDoOutput(true);
327      }
328      return conn;
329    } catch (Exception ex) {
330      throw new IOException(ex);
331    }
332  }
333
334  /**
335   * Called after a new FileSystem instance is constructed.
336   *
337   * @param name a uri whose authority section names the host, port, etc. for this FileSystem
338   * @param conf the configuration
339   */
340  @Override
341  public void initialize(URI name, Configuration conf) throws IOException {
342    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
343
344    //the real use is the one that has the Kerberos credentials needed for
345    //SPNEGO to work
346    realUser = ugi.getRealUser();
347    if (realUser == null) {
348      realUser = UserGroupInformation.getLoginUser();
349    }
350    super.initialize(name, conf);
351    try {
352      uri = new URI(name.getScheme() + "://" + name.getAuthority());
353    } catch (URISyntaxException ex) {
354      throw new IOException(ex);
355    }
356
357    Class<? extends DelegationTokenAuthenticator> klass =
358        getConf().getClass("httpfs.authenticator.class",
359            KerberosDelegationTokenAuthenticator.class,
360            DelegationTokenAuthenticator.class);
361    DelegationTokenAuthenticator authenticator =
362        ReflectionUtils.newInstance(klass, getConf());
363    authURL = new DelegationTokenAuthenticatedURL(authenticator);
364  }
365
366  @Override
367  public String getScheme() {
368    return SCHEME;
369  }
370
371  /**
372   * Returns a URI whose scheme and authority identify this FileSystem.
373   *
374   * @return the URI whose scheme and authority identify this FileSystem.
375   */
376  @Override
377  public URI getUri() {
378    return uri;
379  }
380
381  /**
382   * Get the default port for this file system.
383   * @return the default port or 0 if there isn't one
384   */
385  @Override
386  protected int getDefaultPort() {
387    return getConf().getInt(HdfsClientConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
388        DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
389  }
390
391  /**
392   * HttpFSServer subclass of the <code>FSDataInputStream</code>.
393   * <p>
394   * This implementation does not support the
395   * <code>PositionReadable</code> and <code>Seekable</code> methods.
396   */
397  private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable {
398
399    protected HttpFSDataInputStream(InputStream in, int bufferSize) {
400      super(new BufferedInputStream(in, bufferSize));
401    }
402
403    @Override
404    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
405      throw new UnsupportedOperationException();
406    }
407
408    @Override
409    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
410      throw new UnsupportedOperationException();
411    }
412
413    @Override
414    public void readFully(long position, byte[] buffer) throws IOException {
415      throw new UnsupportedOperationException();
416    }
417
418    @Override
419    public void seek(long pos) throws IOException {
420      throw new UnsupportedOperationException();
421    }
422
423    @Override
424    public long getPos() throws IOException {
425      throw new UnsupportedOperationException();
426    }
427
428    @Override
429    public boolean seekToNewSource(long targetPos) throws IOException {
430      throw new UnsupportedOperationException();
431    }
432  }
433
434  /**
435   * Opens an FSDataInputStream at the indicated Path.
436   * <p>
437   * IMPORTANT: the returned <code>FSDataInputStream</code> does not support the
438   * <code>PositionReadable</code> and <code>Seekable</code> methods.
439   *
440   * @param f the file name to open
441   * @param bufferSize the size of the buffer to be used.
442   */
443  @Override
444  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
445    Map<String, String> params = new HashMap<String, String>();
446    params.put(OP_PARAM, Operation.OPEN.toString());
447    HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params,
448                                           f, true);
449    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
450    return new FSDataInputStream(
451      new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
452  }
453
454  /**
455   * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
456   * <p>
457   * This implementation closes the underlying HTTP connection validating the Http connection status
458   * at closing time.
459   */
460  private static class HttpFSDataOutputStream extends FSDataOutputStream {
461    private HttpURLConnection conn;
462    private int closeStatus;
463
464    public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats)
465      throws IOException {
466      super(out, stats);
467      this.conn = conn;
468      this.closeStatus = closeStatus;
469    }
470
471    @Override
472    public void close() throws IOException {
473      try {
474        super.close();
475      } finally {
476        HttpExceptionUtils.validateResponse(conn, closeStatus);
477      }
478    }
479
480  }
481
482  /**
483   * Converts a <code>FsPermission</code> to a Unix octal representation.
484   *
485   * @param p the permission.
486   *
487   * @return the Unix string symbolic reprentation.
488   */
489  public static String permissionToString(FsPermission p) {
490    return  Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8);
491  }
492
493  /*
494   * Common handling for uploading data for create and append operations.
495   */
496  private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params,
497                                        int bufferSize, int expectedStatus) throws IOException {
498    HttpURLConnection conn = getConnection(method, params, f, true);
499    conn.setInstanceFollowRedirects(false);
500    boolean exceptionAlreadyHandled = false;
501    try {
502      if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) {
503        exceptionAlreadyHandled = true;
504        String location = conn.getHeaderField("Location");
505        if (location != null) {
506          conn = getConnection(new URL(location), method);
507          conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE);
508          try {
509            OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
510            return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
511          } catch (IOException ex) {
512            HttpExceptionUtils.validateResponse(conn, expectedStatus);
513            throw ex;
514          }
515        } else {
516          HttpExceptionUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
517          throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
518        }
519      } else {
520        throw new IOException(
521          MessageFormat.format("Expected HTTP status was [307], received [{0}]",
522                               conn.getResponseCode()));
523      }
524    } catch (IOException ex) {
525      if (exceptionAlreadyHandled) {
526        throw ex;
527      } else {
528        HttpExceptionUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
529        throw ex;
530      }
531    }
532  }
533
534
535  /**
536   * Opens an FSDataOutputStream at the indicated Path with write-progress
537   * reporting.
538   * <p>
539   * IMPORTANT: The <code>Progressable</code> parameter is not used.
540   *
541   * @param f the file name to open.
542   * @param permission file permission.
543   * @param overwrite if a file with this name already exists, then if true,
544   * the file will be overwritten, and if false an error will be thrown.
545   * @param bufferSize the size of the buffer to be used.
546   * @param replication required block replication for the file.
547   * @param blockSize block size.
548   * @param progress progressable.
549   *
550   * @throws IOException
551   * @see #setPermission(Path, FsPermission)
552   */
553  @Override
554  public FSDataOutputStream create(Path f, FsPermission permission,
555                                   boolean overwrite, int bufferSize,
556                                   short replication, long blockSize,
557                                   Progressable progress) throws IOException {
558    Map<String, String> params = new HashMap<String, String>();
559    params.put(OP_PARAM, Operation.CREATE.toString());
560    params.put(OVERWRITE_PARAM, Boolean.toString(overwrite));
561    params.put(REPLICATION_PARAM, Short.toString(replication));
562    params.put(BLOCKSIZE_PARAM, Long.toString(blockSize));
563    params.put(PERMISSION_PARAM, permissionToString(permission));
564    return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize,
565                      HttpURLConnection.HTTP_CREATED);
566  }
567
568
569  /**
570   * Append to an existing file (optional operation).
571   * <p>
572   * IMPORTANT: The <code>Progressable</code> parameter is not used.
573   *
574   * @param f the existing file to be appended.
575   * @param bufferSize the size of the buffer to be used.
576   * @param progress for reporting progress if it is not null.
577   *
578   * @throws IOException
579   */
580  @Override
581  public FSDataOutputStream append(Path f, int bufferSize,
582                                   Progressable progress) throws IOException {
583    Map<String, String> params = new HashMap<String, String>();
584    params.put(OP_PARAM, Operation.APPEND.toString());
585    return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize,
586                      HttpURLConnection.HTTP_OK);
587  }
588
589  /**
590   * Truncate a file.
591   * 
592   * @param f the file to be truncated.
593   * @param newLength The size the file is to be truncated to.
594   *
595   * @throws IOException
596   */
597  @Override
598  public boolean truncate(Path f, long newLength) throws IOException {
599    Map<String, String> params = new HashMap<String, String>();
600    params.put(OP_PARAM, Operation.TRUNCATE.toString());
601    params.put(NEW_LENGTH_PARAM, Long.toString(newLength));
602    HttpURLConnection conn = getConnection(Operation.TRUNCATE.getMethod(),
603        params, f, true);
604    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
605    return (Boolean) json.get(TRUNCATE_JSON);
606  }
607
608  /**
609   * Concat existing files together.
610   * @param f the path to the target destination.
611   * @param psrcs the paths to the sources to use for the concatenation.
612   *
613   * @throws IOException
614   */
615  @Override
616  public void concat(Path f, Path[] psrcs) throws IOException {
617    List<String> strPaths = new ArrayList<String>(psrcs.length);
618    for(Path psrc : psrcs) {
619      strPaths.add(psrc.toUri().getPath());
620    }
621    String srcs = StringUtils.join(",", strPaths);
622
623    Map<String, String> params = new HashMap<String, String>();
624    params.put(OP_PARAM, Operation.CONCAT.toString());
625    params.put(SOURCES_PARAM, srcs);
626    HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
627        params, f, true);
628    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
629  }
630
631  /**
632   * Renames Path src to Path dst.  Can take place on local fs
633   * or remote DFS.
634   */
635  @Override
636  public boolean rename(Path src, Path dst) throws IOException {
637    Map<String, String> params = new HashMap<String, String>();
638    params.put(OP_PARAM, Operation.RENAME.toString());
639    params.put(DESTINATION_PARAM, dst.toString());
640    HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(),
641                                           params, src, true);
642    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
643    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
644    return (Boolean) json.get(RENAME_JSON);
645  }
646
647  /**
648   * Delete a file.
649   *
650   * @deprecated Use delete(Path, boolean) instead
651   */
652  @Deprecated
653  @Override
654  public boolean delete(Path f) throws IOException {
655    return delete(f, false);
656  }
657
658  /**
659   * Delete a file.
660   *
661   * @param f the path to delete.
662   * @param recursive if path is a directory and set to
663   * true, the directory is deleted else throws an exception. In
664   * case of a file the recursive can be set to either true or false.
665   *
666   * @return true if delete is successful else false.
667   *
668   * @throws IOException
669   */
670  @Override
671  public boolean delete(Path f, boolean recursive) throws IOException {
672    Map<String, String> params = new HashMap<String, String>();
673    params.put(OP_PARAM, Operation.DELETE.toString());
674    params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
675    HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(),
676                                           params, f, true);
677    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
678    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
679    return (Boolean) json.get(DELETE_JSON);
680  }
681
682  /**
683   * List the statuses of the files/directories in the given path if the path is
684   * a directory.
685   *
686   * @param f given path
687   *
688   * @return the statuses of the files/directories in the given patch
689   *
690   * @throws IOException
691   */
692  @Override
693  public FileStatus[] listStatus(Path f) throws IOException {
694    Map<String, String> params = new HashMap<String, String>();
695    params.put(OP_PARAM, Operation.LISTSTATUS.toString());
696    HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(),
697                                           params, f, true);
698    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
699    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
700    json = (JSONObject) json.get(FILE_STATUSES_JSON);
701    JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
702    FileStatus[] array = new FileStatus[jsonArray.size()];
703    f = makeQualified(f);
704    for (int i = 0; i < jsonArray.size(); i++) {
705      array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i));
706    }
707    return array;
708  }
709
710  /**
711   * Set the current working directory for the given file system. All relative
712   * paths will be resolved relative to it.
713   *
714   * @param newDir new directory.
715   */
716  @Override
717  public void setWorkingDirectory(Path newDir) {
718    workingDir = newDir;
719  }
720
721  /**
722   * Get the current working directory for the given file system
723   *
724   * @return the directory pathname
725   */
726  @Override
727  public Path getWorkingDirectory() {
728    if (workingDir == null) {
729      workingDir = getHomeDirectory();
730    }
731    return workingDir;
732  }
733
734  /**
735   * Make the given file and all non-existent parents into
736   * directories. Has the semantics of Unix 'mkdir -p'.
737   * Existence of the directory hierarchy is not an error.
738   */
739  @Override
740  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
741    Map<String, String> params = new HashMap<String, String>();
742    params.put(OP_PARAM, Operation.MKDIRS.toString());
743    params.put(PERMISSION_PARAM, permissionToString(permission));
744    HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(),
745                                           params, f, true);
746    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
747    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
748    return (Boolean) json.get(MKDIRS_JSON);
749  }
750
751  /**
752   * Return a file status object that represents the path.
753   *
754   * @param f The path we want information from
755   *
756   * @return a FileStatus object
757   *
758   * @throws FileNotFoundException when the path does not exist;
759   * IOException see specific implementation
760   */
761  @Override
762  public FileStatus getFileStatus(Path f) throws IOException {
763    Map<String, String> params = new HashMap<String, String>();
764    params.put(OP_PARAM, Operation.GETFILESTATUS.toString());
765    HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(),
766                                           params, f, true);
767    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
768    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
769    json = (JSONObject) json.get(FILE_STATUS_JSON);
770    f = makeQualified(f);
771    return createFileStatus(f, json);
772  }
773
774  /**
775   * Return the current user's home directory in this filesystem.
776   * The default implementation returns "/user/$USER/".
777   */
778  @Override
779  public Path getHomeDirectory() {
780    Map<String, String> params = new HashMap<String, String>();
781    params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString());
782    try {
783      HttpURLConnection conn =
784        getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params,
785                      new Path(getUri().toString(), "/"), false);
786      HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
787      JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
788      return new Path((String) json.get(HOME_DIR_JSON));
789    } catch (IOException ex) {
790      throw new RuntimeException(ex);
791    }
792  }
793
794  /**
795   * Set owner of a path (i.e. a file or a directory).
796   * The parameters username and groupname cannot both be null.
797   *
798   * @param p The path
799   * @param username If it is null, the original username remains unchanged.
800   * @param groupname If it is null, the original groupname remains unchanged.
801   */
802  @Override
803  public void setOwner(Path p, String username, String groupname)
804    throws IOException {
805    Map<String, String> params = new HashMap<String, String>();
806    params.put(OP_PARAM, Operation.SETOWNER.toString());
807    params.put(OWNER_PARAM, username);
808    params.put(GROUP_PARAM, groupname);
809    HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(),
810                                           params, p, true);
811    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
812  }
813
814  /**
815   * Set permission of a path.
816   *
817   * @param p path.
818   * @param permission permission.
819   */
820  @Override
821  public void setPermission(Path p, FsPermission permission) throws IOException {
822    Map<String, String> params = new HashMap<String, String>();
823    params.put(OP_PARAM, Operation.SETPERMISSION.toString());
824    params.put(PERMISSION_PARAM, permissionToString(permission));
825    HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true);
826    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
827  }
828
829  /**
830   * Set access time of a file
831   *
832   * @param p The path
833   * @param mtime Set the modification time of this file.
834   * The number of milliseconds since Jan 1, 1970.
835   * A value of -1 means that this call should not set modification time.
836   * @param atime Set the access time of this file.
837   * The number of milliseconds since Jan 1, 1970.
838   * A value of -1 means that this call should not set access time.
839   */
840  @Override
841  public void setTimes(Path p, long mtime, long atime) throws IOException {
842    Map<String, String> params = new HashMap<String, String>();
843    params.put(OP_PARAM, Operation.SETTIMES.toString());
844    params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime));
845    params.put(ACCESS_TIME_PARAM, Long.toString(atime));
846    HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(),
847                                           params, p, true);
848    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
849  }
850
851  /**
852   * Set replication for an existing file.
853   *
854   * @param src file name
855   * @param replication new replication
856   *
857   * @return true if successful;
858   *         false if file does not exist or is a directory
859   *
860   * @throws IOException
861   */
862  @Override
863  public boolean setReplication(Path src, short replication)
864    throws IOException {
865    Map<String, String> params = new HashMap<String, String>();
866    params.put(OP_PARAM, Operation.SETREPLICATION.toString());
867    params.put(REPLICATION_PARAM, Short.toString(replication));
868    HttpURLConnection conn =
869      getConnection(Operation.SETREPLICATION.getMethod(), params, src, true);
870    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
871    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
872    return (Boolean) json.get(SET_REPLICATION_JSON);
873  }
874
875  /**
876   * Modify the ACL entries for a file.
877   *
878   * @param path Path to modify
879   * @param aclSpec describing modifications
880   * @throws IOException
881   */
882  @Override
883  public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
884          throws IOException {
885    Map<String, String> params = new HashMap<String, String>();
886    params.put(OP_PARAM, Operation.MODIFYACLENTRIES.toString());
887    params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec));
888    HttpURLConnection conn = getConnection(
889            Operation.MODIFYACLENTRIES.getMethod(), params, path, true);
890    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
891  }
892
893  /**
894   * Remove the specified ACL entries from a file
895   * @param path Path to modify
896   * @param aclSpec describing entries to remove
897   * @throws IOException
898   */
899  @Override
900  public void removeAclEntries(Path path, List<AclEntry> aclSpec)
901          throws IOException {
902    Map<String, String> params = new HashMap<String, String>();
903    params.put(OP_PARAM, Operation.REMOVEACLENTRIES.toString());
904    params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec));
905    HttpURLConnection conn = getConnection(
906            Operation.REMOVEACLENTRIES.getMethod(), params, path, true);
907    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
908  }
909
910  /**
911   * Removes the default ACL for the given file
912   * @param path Path from which to remove the default ACL.
913   * @throws IOException
914   */
915  @Override
916  public void removeDefaultAcl(Path path) throws IOException {
917    Map<String, String> params = new HashMap<String, String>();
918    params.put(OP_PARAM, Operation.REMOVEDEFAULTACL.toString());
919    HttpURLConnection conn = getConnection(
920            Operation.REMOVEDEFAULTACL.getMethod(), params, path, true);
921    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
922  }
923
924  /**
925   * Remove all ACLs from a file
926   * @param path Path from which to remove all ACLs
927   * @throws IOException
928   */
929  @Override
930  public void removeAcl(Path path) throws IOException {
931    Map<String, String> params = new HashMap<String, String>();
932    params.put(OP_PARAM, Operation.REMOVEACL.toString());
933    HttpURLConnection conn = getConnection(Operation.REMOVEACL.getMethod(),
934            params, path, true);
935    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
936  }
937
938  /**
939   * Set the ACLs for the given file
940   * @param path Path to modify
941   * @param aclSpec describing modifications, must include
942   *                entries for user, group, and others for compatibility
943   *                with permission bits.
944   * @throws IOException
945   */
946  @Override
947  public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
948    Map<String, String> params = new HashMap<String, String>();
949    params.put(OP_PARAM, Operation.SETACL.toString());
950    params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec));
951    HttpURLConnection conn = getConnection(Operation.SETACL.getMethod(),
952                                           params, path, true);
953    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
954  }
955
956  /**
957   * Get the ACL information for a given file
958   * @param path Path to acquire ACL info for
959   * @return the ACL information in JSON format
960   * @throws IOException
961   */
962  @Override
963  public AclStatus getAclStatus(Path path) throws IOException {
964    Map<String, String> params = new HashMap<String, String>();
965    params.put(OP_PARAM, Operation.GETACLSTATUS.toString());
966    HttpURLConnection conn = getConnection(Operation.GETACLSTATUS.getMethod(),
967            params, path, true);
968    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
969    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
970    json = (JSONObject) json.get(ACL_STATUS_JSON);
971    return createAclStatus(json);
972  }
973
974  /** Convert a string to a FsPermission object. */
975  static FsPermission toFsPermission(JSONObject json) {
976    final String s = (String) json.get(PERMISSION_JSON);
977    final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
978    final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
979    FsPermission perm = new FsPermission(Short.parseShort(s, 8));
980    final boolean aBit = (aclBit != null) ? aclBit : false;
981    final boolean eBit = (encBit != null) ? encBit : false;
982    if (aBit || eBit) {
983      return new FsPermissionExtension(perm, aBit, eBit);
984    } else {
985      return perm;
986    }
987  }
988
989  private FileStatus createFileStatus(Path parent, JSONObject json) {
990    String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
991    Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
992    FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON));
993    long len = (Long) json.get(LENGTH_JSON);
994    String owner = (String) json.get(OWNER_JSON);
995    String group = (String) json.get(GROUP_JSON);
996    final FsPermission permission = toFsPermission(json);
997    long aTime = (Long) json.get(ACCESS_TIME_JSON);
998    long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
999    long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
1000    short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
1001    FileStatus fileStatus = null;
1002
1003    switch (type) {
1004      case FILE:
1005      case DIRECTORY:
1006        fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
1007                                    replication, blockSize, mTime, aTime,
1008                                    permission, owner, group, path);
1009        break;
1010      case SYMLINK:
1011        Path symLink = null;
1012        fileStatus = new FileStatus(len, false,
1013                                    replication, blockSize, mTime, aTime,
1014                                    permission, owner, group, symLink,
1015                                    path);
1016    }
1017    return fileStatus;
1018  }
1019
1020  /**
1021   * Convert the given JSON object into an AclStatus
1022   * @param json Input JSON representing the ACLs
1023   * @return Resulting AclStatus
1024   */
1025  private AclStatus createAclStatus(JSONObject json) {
1026    AclStatus.Builder aclStatusBuilder = new AclStatus.Builder()
1027            .owner((String) json.get(OWNER_JSON))
1028            .group((String) json.get(GROUP_JSON))
1029            .stickyBit((Boolean) json.get(ACL_STICKY_BIT_JSON));
1030    JSONArray entries = (JSONArray) json.get(ACL_ENTRIES_JSON);
1031    for ( Object e : entries ) {
1032      aclStatusBuilder.addEntry(AclEntry.parseAclEntry(e.toString(), true));
1033    }
1034    return aclStatusBuilder.build();
1035  }
1036
1037  @Override
1038  public ContentSummary getContentSummary(Path f) throws IOException {
1039    Map<String, String> params = new HashMap<String, String>();
1040    params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString());
1041    HttpURLConnection conn =
1042      getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true);
1043    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1044    JSONObject json = (JSONObject) ((JSONObject)
1045      HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
1046    return new ContentSummary.Builder().
1047        length((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON)).
1048        fileCount((Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON)).
1049        directoryCount((Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON)).
1050        quota((Long) json.get(CONTENT_SUMMARY_QUOTA_JSON)).
1051        spaceConsumed((Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON)).
1052        spaceQuota((Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)).build();
1053  }
1054
1055  @Override
1056  public FileChecksum getFileChecksum(Path f) throws IOException {
1057    Map<String, String> params = new HashMap<String, String>();
1058    params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString());
1059    HttpURLConnection conn =
1060      getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true);
1061    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1062    final JSONObject json = (JSONObject) ((JSONObject)
1063      HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON);
1064    return new FileChecksum() {
1065      @Override
1066      public String getAlgorithmName() {
1067        return (String) json.get(CHECKSUM_ALGORITHM_JSON);
1068      }
1069
1070      @Override
1071      public int getLength() {
1072        return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue();
1073      }
1074
1075      @Override
1076      public byte[] getBytes() {
1077        return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON));
1078      }
1079
1080      @Override
1081      public void write(DataOutput out) throws IOException {
1082        throw new UnsupportedOperationException();
1083      }
1084
1085      @Override
1086      public void readFields(DataInput in) throws IOException {
1087        throw new UnsupportedOperationException();
1088      }
1089    };
1090  }
1091
1092
1093  @Override
1094  public Token<?> getDelegationToken(final String renewer)
1095    throws IOException {
1096    try {
1097      return UserGroupInformation.getCurrentUser().doAs(
1098          new PrivilegedExceptionAction<Token<?>>() {
1099            @Override
1100            public Token<?> run() throws Exception {
1101              return authURL.getDelegationToken(uri.toURL(), authToken,
1102                  renewer);
1103            }
1104          }
1105      );
1106    } catch (Exception ex) {
1107      if (ex instanceof IOException) {
1108        throw (IOException) ex;
1109      } else {
1110        throw new IOException(ex);
1111      }
1112    }
1113  }
1114
1115  public long renewDelegationToken(final Token<?> token) throws IOException {
1116    try {
1117      return UserGroupInformation.getCurrentUser().doAs(
1118          new PrivilegedExceptionAction<Long>() {
1119            @Override
1120            public Long run() throws Exception {
1121              return authURL.renewDelegationToken(uri.toURL(), authToken);
1122            }
1123          }
1124      );
1125    } catch (Exception ex) {
1126      if (ex instanceof IOException) {
1127        throw (IOException) ex;
1128      } else {
1129        throw new IOException(ex);
1130      }
1131    }
1132  }
1133
1134  public void cancelDelegationToken(final Token<?> token) throws IOException {
1135    authURL.cancelDelegationToken(uri.toURL(), authToken);
1136  }
1137
1138  @Override
1139  public Token<?> getRenewToken() {
1140    return null; //TODO : for renewer
1141  }
1142
1143  @Override
1144  @SuppressWarnings("unchecked")
1145  public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
1146    //TODO : for renewer
1147  }
1148
1149  @Override
1150  public void setXAttr(Path f, String name, byte[] value,
1151      EnumSet<XAttrSetFlag> flag) throws IOException {
1152    Map<String, String> params = new HashMap<String, String>();
1153    params.put(OP_PARAM, Operation.SETXATTR.toString());
1154    params.put(XATTR_NAME_PARAM, name);
1155    if (value != null) {
1156      params.put(XATTR_VALUE_PARAM, 
1157          XAttrCodec.encodeValue(value, XAttrCodec.HEX));
1158    }
1159    params.put(XATTR_SET_FLAG_PARAM, EnumSetParam.toString(flag));
1160    HttpURLConnection conn = getConnection(Operation.SETXATTR.getMethod(),
1161        params, f, true);
1162    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1163  }
1164
1165  @Override
1166  public byte[] getXAttr(Path f, String name) throws IOException {
1167    Map<String, String> params = new HashMap<String, String>();
1168    params.put(OP_PARAM, Operation.GETXATTRS.toString());
1169    params.put(XATTR_NAME_PARAM, name);
1170    HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(),
1171        params, f, true);
1172    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1173    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1174    Map<String, byte[]> xAttrs = createXAttrMap(
1175        (JSONArray) json.get(XATTRS_JSON));
1176    return xAttrs != null ? xAttrs.get(name) : null;
1177  }
1178
1179  /** Convert xAttrs json to xAttrs map */
1180  private Map<String, byte[]> createXAttrMap(JSONArray jsonArray) 
1181      throws IOException {
1182    Map<String, byte[]> xAttrs = Maps.newHashMap();
1183    for (Object obj : jsonArray) {
1184      JSONObject jsonObj = (JSONObject) obj;
1185      final String name = (String)jsonObj.get(XATTR_NAME_JSON);
1186      final byte[] value = XAttrCodec.decodeValue(
1187          (String)jsonObj.get(XATTR_VALUE_JSON));
1188      xAttrs.put(name, value);
1189    }
1190
1191    return xAttrs;
1192  }
1193
1194  /** Convert xAttr names json to names list */
1195  private List<String> createXAttrNames(String xattrNamesStr) throws IOException {
1196    JSONParser parser = new JSONParser();
1197    JSONArray jsonArray;
1198    try {
1199      jsonArray = (JSONArray)parser.parse(xattrNamesStr);
1200      List<String> names = Lists.newArrayListWithCapacity(jsonArray.size());
1201      for (Object name : jsonArray) {
1202        names.add((String) name);
1203      }
1204      return names;
1205    } catch (ParseException e) {
1206      throw new IOException("JSON parser error, " + e.getMessage(), e);
1207    }
1208  }
1209
1210  @Override
1211  public Map<String, byte[]> getXAttrs(Path f) throws IOException {
1212    Map<String, String> params = new HashMap<String, String>();
1213    params.put(OP_PARAM, Operation.GETXATTRS.toString());
1214    HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(),
1215        params, f, true);
1216    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1217    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1218    return createXAttrMap((JSONArray) json.get(XATTRS_JSON));
1219  }
1220
1221  @Override
1222  public Map<String, byte[]> getXAttrs(Path f, List<String> names)
1223      throws IOException {
1224    Preconditions.checkArgument(names != null && !names.isEmpty(), 
1225        "XAttr names cannot be null or empty.");
1226    Map<String, String> params = new HashMap<String, String>();
1227    params.put(OP_PARAM, Operation.GETXATTRS.toString());
1228    Map<String, List<String>> multiValuedParams = Maps.newHashMap();
1229    multiValuedParams.put(XATTR_NAME_PARAM, names);
1230    HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(),
1231        params, multiValuedParams, f, true);
1232    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1233    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1234    return createXAttrMap((JSONArray) json.get(XATTRS_JSON));
1235  }
1236
1237  @Override
1238  public List<String> listXAttrs(Path f) throws IOException {
1239    Map<String, String> params = new HashMap<String, String>();
1240    params.put(OP_PARAM, Operation.LISTXATTRS.toString());
1241    HttpURLConnection conn = getConnection(Operation.LISTXATTRS.getMethod(),
1242        params, f, true);
1243    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1244    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1245    return createXAttrNames((String) json.get(XATTRNAMES_JSON));
1246  }
1247
1248  @Override
1249  public void removeXAttr(Path f, String name) throws IOException {
1250    Map<String, String> params = new HashMap<String, String>();
1251    params.put(OP_PARAM, Operation.REMOVEXATTR.toString());
1252    params.put(XATTR_NAME_PARAM, name);
1253    HttpURLConnection conn = getConnection(Operation.REMOVEXATTR.getMethod(),
1254        params, f, true);
1255    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1256  }
1257
1258  @Override
1259  public Collection<BlockStoragePolicy> getAllStoragePolicies()
1260      throws IOException {
1261    Map<String, String> params = new HashMap<String, String>();
1262    params.put(OP_PARAM, Operation.GETALLSTORAGEPOLICY.toString());
1263    HttpURLConnection conn = getConnection(
1264        Operation.GETALLSTORAGEPOLICY.getMethod(), params, new Path(getUri()
1265            .toString(), "/"), false);
1266    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1267    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1268    return createStoragePolicies((JSONObject) json.get(STORAGE_POLICIES_JSON));
1269  }
1270
1271  private Collection<BlockStoragePolicy> createStoragePolicies(JSONObject map)
1272      throws IOException {
1273    JSONArray jsonArray = (JSONArray) map.get(STORAGE_POLICY_JSON);
1274    BlockStoragePolicy[] policies = new BlockStoragePolicy[jsonArray.size()];
1275    for (int i = 0; i < jsonArray.size(); i++) {
1276      policies[i] = createStoragePolicy((JSONObject) jsonArray.get(i));
1277    }
1278    return Arrays.asList(policies);
1279  }
1280
1281  @Override
1282  public BlockStoragePolicy getStoragePolicy(Path src) throws IOException {
1283    Map<String, String> params = new HashMap<String, String>();
1284    params.put(OP_PARAM, Operation.GETSTORAGEPOLICY.toString());
1285    HttpURLConnection conn = getConnection(
1286        Operation.GETSTORAGEPOLICY.getMethod(), params, src, true);
1287    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1288    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
1289    return createStoragePolicy((JSONObject) json.get(STORAGE_POLICY_JSON));
1290  }
1291  private BlockStoragePolicy createStoragePolicy(JSONObject policyJson)
1292      throws IOException {
1293    byte id = ((Number) policyJson.get("id")).byteValue();
1294    String name = (String) policyJson.get("name");
1295    StorageType[] storageTypes = toStorageTypes((JSONArray) policyJson
1296        .get("storageTypes"));
1297    StorageType[] creationFallbacks = toStorageTypes((JSONArray) policyJson
1298        .get("creationFallbacks"));
1299    StorageType[] replicationFallbacks = toStorageTypes((JSONArray) policyJson
1300        .get("replicationFallbacks"));
1301    Boolean copyOnCreateFile = (Boolean) policyJson.get("copyOnCreateFile");
1302    return new BlockStoragePolicy(id, name, storageTypes, creationFallbacks,
1303        replicationFallbacks, copyOnCreateFile.booleanValue());
1304  }
1305
1306  private StorageType[] toStorageTypes(JSONArray array) throws IOException {
1307    if (array == null) {
1308      return null;
1309    } else {
1310      List<StorageType> storageTypes = new ArrayList<StorageType>(array.size());
1311      for (Object name : array) {
1312        storageTypes.add(StorageType.parseStorageType((String) name));
1313      }
1314      return storageTypes.toArray(new StorageType[storageTypes.size()]);
1315    }
1316  }
1317
1318  @Override
1319  public void setStoragePolicy(Path src, String policyName) throws IOException {
1320    Map<String, String> params = new HashMap<String, String>();
1321    params.put(OP_PARAM, Operation.SETSTORAGEPOLICY.toString());
1322    params.put(POLICY_NAME_PARAM, policyName);
1323    HttpURLConnection conn = getConnection(
1324        Operation.SETSTORAGEPOLICY.getMethod(), params, src, true);
1325    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1326  }
1327
1328  @Override
1329  public void unsetStoragePolicy(Path src) throws IOException {
1330    Map<String, String> params = new HashMap<String, String>();
1331    params.put(OP_PARAM, Operation.UNSETSTORAGEPOLICY.toString());
1332    HttpURLConnection conn = getConnection(
1333        Operation.UNSETSTORAGEPOLICY.getMethod(), params, src, true);
1334    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
1335  }
1336}