001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.fs.http.client; 019 020import java.util.ArrayList; 021import java.util.Arrays; 022import java.util.Collection; 023import java.util.EnumSet; 024import java.util.List; 025 026import org.apache.hadoop.classification.InterfaceAudience; 027import org.apache.hadoop.conf.Configuration; 028import org.apache.hadoop.fs.ContentSummary; 029import org.apache.hadoop.fs.DelegationTokenRenewer; 030import org.apache.hadoop.fs.FSDataInputStream; 031import org.apache.hadoop.fs.FSDataOutputStream; 032import org.apache.hadoop.fs.FileChecksum; 033import org.apache.hadoop.fs.FileStatus; 034import org.apache.hadoop.fs.FileSystem; 035import org.apache.hadoop.fs.Path; 036import org.apache.hadoop.fs.PositionedReadable; 037import org.apache.hadoop.fs.Seekable; 038import org.apache.hadoop.fs.StorageType; 039import org.apache.hadoop.fs.XAttrCodec; 040import org.apache.hadoop.fs.XAttrSetFlag; 041import org.apache.hadoop.fs.permission.AclEntry; 042import org.apache.hadoop.fs.permission.AclStatus; 043import org.apache.hadoop.fs.permission.FsPermission; 044import org.apache.hadoop.hdfs.DFSConfigKeys; 045import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; 046import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; 047import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; 048import org.apache.hadoop.lib.wsrs.EnumSetParam; 049import org.apache.hadoop.security.UserGroupInformation; 050import org.apache.hadoop.security.token.Token; 051import org.apache.hadoop.security.token.TokenIdentifier; 052import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; 053import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; 054import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; 055import org.apache.hadoop.util.HttpExceptionUtils; 056import org.apache.hadoop.util.Progressable; 057import org.apache.hadoop.util.ReflectionUtils; 058import org.apache.hadoop.util.StringUtils; 059import org.json.simple.JSONArray; 060import org.json.simple.JSONObject; 061import org.json.simple.parser.JSONParser; 062import org.json.simple.parser.ParseException; 063 064import com.google.common.base.Preconditions; 065import com.google.common.collect.Lists; 066import com.google.common.collect.Maps; 067 068import java.io.BufferedInputStream; 069import java.io.BufferedOutputStream; 070import java.io.DataInput; 071import java.io.DataOutput; 072import java.io.FileNotFoundException; 073import java.io.FilterInputStream; 074import java.io.IOException; 075import java.io.InputStream; 076import java.io.OutputStream; 077import java.net.HttpURLConnection; 078import java.net.URI; 079import java.net.URISyntaxException; 080import java.net.URL; 081import java.security.PrivilegedExceptionAction; 082import java.text.MessageFormat; 083import java.util.HashMap; 084import java.util.Map; 085 086/** 087 * HttpFSServer implementation of the FileSystemAccess FileSystem. 088 * <p> 089 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. 090 */ 091@InterfaceAudience.Private 092public class HttpFSFileSystem extends FileSystem 093 implements DelegationTokenRenewer.Renewable { 094 095 public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME; 096 097 public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION; 098 099 public static final String SCHEME = "webhdfs"; 100 101 public static final String OP_PARAM = "op"; 102 public static final String DO_AS_PARAM = "doas"; 103 public static final String OVERWRITE_PARAM = "overwrite"; 104 public static final String REPLICATION_PARAM = "replication"; 105 public static final String BLOCKSIZE_PARAM = "blocksize"; 106 public static final String PERMISSION_PARAM = "permission"; 107 public static final String ACLSPEC_PARAM = "aclspec"; 108 public static final String DESTINATION_PARAM = "destination"; 109 public static final String RECURSIVE_PARAM = "recursive"; 110 public static final String SOURCES_PARAM = "sources"; 111 public static final String OWNER_PARAM = "owner"; 112 public static final String GROUP_PARAM = "group"; 113 public static final String MODIFICATION_TIME_PARAM = "modificationtime"; 114 public static final String ACCESS_TIME_PARAM = "accesstime"; 115 public static final String XATTR_NAME_PARAM = "xattr.name"; 116 public static final String XATTR_VALUE_PARAM = "xattr.value"; 117 public static final String XATTR_SET_FLAG_PARAM = "flag"; 118 public static final String XATTR_ENCODING_PARAM = "encoding"; 119 public static final String NEW_LENGTH_PARAM = "newlength"; 120 public static final String POLICY_NAME_PARAM = "storagepolicy"; 121 122 public static final Short DEFAULT_PERMISSION = 0755; 123 public static final String ACLSPEC_DEFAULT = ""; 124 125 public static final String RENAME_JSON = "boolean"; 126 127 public static final String TRUNCATE_JSON = "boolean"; 128 129 public static final String DELETE_JSON = "boolean"; 130 131 public static final String MKDIRS_JSON = "boolean"; 132 133 public static final String HOME_DIR_JSON = "Path"; 134 135 public static final String SET_REPLICATION_JSON = "boolean"; 136 137 public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; 138 139 public static enum FILE_TYPE { 140 FILE, DIRECTORY, SYMLINK; 141 142 public static FILE_TYPE getType(FileStatus fileStatus) { 143 if (fileStatus.isFile()) { 144 return FILE; 145 } 146 if (fileStatus.isDirectory()) { 147 return DIRECTORY; 148 } 149 if (fileStatus.isSymlink()) { 150 return SYMLINK; 151 } 152 throw new IllegalArgumentException("Could not determine filetype for: " + 153 fileStatus.getPath()); 154 } 155 } 156 157 public static final String FILE_STATUSES_JSON = "FileStatuses"; 158 public static final String FILE_STATUS_JSON = "FileStatus"; 159 public static final String PATH_SUFFIX_JSON = "pathSuffix"; 160 public static final String TYPE_JSON = "type"; 161 public static final String LENGTH_JSON = "length"; 162 public static final String OWNER_JSON = "owner"; 163 public static final String GROUP_JSON = "group"; 164 public static final String PERMISSION_JSON = "permission"; 165 public static final String ACCESS_TIME_JSON = "accessTime"; 166 public static final String MODIFICATION_TIME_JSON = "modificationTime"; 167 public static final String BLOCK_SIZE_JSON = "blockSize"; 168 public static final String REPLICATION_JSON = "replication"; 169 public static final String XATTRS_JSON = "XAttrs"; 170 public static final String XATTR_NAME_JSON = "name"; 171 public static final String XATTR_VALUE_JSON = "value"; 172 public static final String XATTRNAMES_JSON = "XAttrNames"; 173 174 public static final String FILE_CHECKSUM_JSON = "FileChecksum"; 175 public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; 176 public static final String CHECKSUM_BYTES_JSON = "bytes"; 177 public static final String CHECKSUM_LENGTH_JSON = "length"; 178 179 public static final String CONTENT_SUMMARY_JSON = "ContentSummary"; 180 public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount"; 181 public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount"; 182 public static final String CONTENT_SUMMARY_LENGTH_JSON = "length"; 183 public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota"; 184 public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; 185 public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; 186 187 public static final String ACL_STATUS_JSON = "AclStatus"; 188 public static final String ACL_STICKY_BIT_JSON = "stickyBit"; 189 public static final String ACL_ENTRIES_JSON = "entries"; 190 public static final String ACL_BIT_JSON = "aclBit"; 191 192 public static final String ENC_BIT_JSON = "encBit"; 193 194 public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies"; 195 public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy"; 196 197 public static final int HTTP_TEMPORARY_REDIRECT = 307; 198 199 private static final String HTTP_GET = "GET"; 200 private static final String HTTP_PUT = "PUT"; 201 private static final String HTTP_POST = "POST"; 202 private static final String HTTP_DELETE = "DELETE"; 203 204 @InterfaceAudience.Private 205 public static enum Operation { 206 OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET), 207 GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), 208 GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), 209 INSTRUMENTATION(HTTP_GET), GETACLSTATUS(HTTP_GET), 210 APPEND(HTTP_POST), CONCAT(HTTP_POST), TRUNCATE(HTTP_POST), 211 CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), 212 SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), 213 MODIFYACLENTRIES(HTTP_PUT), REMOVEACLENTRIES(HTTP_PUT), 214 REMOVEDEFAULTACL(HTTP_PUT), REMOVEACL(HTTP_PUT), SETACL(HTTP_PUT), 215 DELETE(HTTP_DELETE), SETXATTR(HTTP_PUT), GETXATTRS(HTTP_GET), 216 REMOVEXATTR(HTTP_PUT), LISTXATTRS(HTTP_GET), 217 GETALLSTORAGEPOLICY(HTTP_GET), GETSTORAGEPOLICY(HTTP_GET), 218 SETSTORAGEPOLICY(HTTP_PUT), UNSETSTORAGEPOLICY(HTTP_POST); 219 220 private String httpMethod; 221 222 Operation(String httpMethod) { 223 this.httpMethod = httpMethod; 224 } 225 226 public String getMethod() { 227 return httpMethod; 228 } 229 230 } 231 232 private DelegationTokenAuthenticatedURL authURL; 233 private DelegationTokenAuthenticatedURL.Token authToken = 234 new DelegationTokenAuthenticatedURL.Token(); 235 private URI uri; 236 private Path workingDir; 237 private UserGroupInformation realUser; 238 239 240 241 /** 242 * Convenience method that creates a <code>HttpURLConnection</code> for the 243 * HttpFSServer file system operations. 244 * <p> 245 * This methods performs and injects any needed authentication credentials 246 * via the {@link #getConnection(URL, String)} method 247 * 248 * @param method the HTTP method. 249 * @param params the query string parameters. 250 * @param path the file path 251 * @param makeQualified if the path should be 'makeQualified' 252 * 253 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, 254 * authenticated and ready to use for the specified path and file system operation. 255 * 256 * @throws IOException thrown if an IO error occurs. 257 */ 258 private HttpURLConnection getConnection(final String method, 259 Map<String, String> params, Path path, boolean makeQualified) 260 throws IOException { 261 return getConnection(method, params, null, path, makeQualified); 262 } 263 264 /** 265 * Convenience method that creates a <code>HttpURLConnection</code> for the 266 * HttpFSServer file system operations. 267 * <p/> 268 * This methods performs and injects any needed authentication credentials 269 * via the {@link #getConnection(URL, String)} method 270 * 271 * @param method the HTTP method. 272 * @param params the query string parameters. 273 * @param multiValuedParams multi valued parameters of the query string 274 * @param path the file path 275 * @param makeQualified if the path should be 'makeQualified' 276 * 277 * @return HttpURLConnection a <code>HttpURLConnection</code> for the 278 * HttpFSServer server, authenticated and ready to use for the 279 * specified path and file system operation. 280 * 281 * @throws IOException thrown if an IO error occurs. 282 */ 283 private HttpURLConnection getConnection(final String method, 284 Map<String, String> params, Map<String, List<String>> multiValuedParams, 285 Path path, boolean makeQualified) throws IOException { 286 if (makeQualified) { 287 path = makeQualified(path); 288 } 289 final URL url = HttpFSUtils.createURL(path, params, multiValuedParams); 290 try { 291 return UserGroupInformation.getCurrentUser().doAs( 292 new PrivilegedExceptionAction<HttpURLConnection>() { 293 @Override 294 public HttpURLConnection run() throws Exception { 295 return getConnection(url, method); 296 } 297 } 298 ); 299 } catch (Exception ex) { 300 if (ex instanceof IOException) { 301 throw (IOException) ex; 302 } else { 303 throw new IOException(ex); 304 } 305 } 306 } 307 308 /** 309 * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL. 310 * <p> 311 * This methods performs and injects any needed authentication credentials. 312 * 313 * @param url url to connect to. 314 * @param method the HTTP method. 315 * 316 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for 317 * the specified path and file system operation. 318 * 319 * @throws IOException thrown if an IO error occurs. 320 */ 321 private HttpURLConnection getConnection(URL url, String method) throws IOException { 322 try { 323 HttpURLConnection conn = authURL.openConnection(url, authToken); 324 conn.setRequestMethod(method); 325 if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { 326 conn.setDoOutput(true); 327 } 328 return conn; 329 } catch (Exception ex) { 330 throw new IOException(ex); 331 } 332 } 333 334 /** 335 * Called after a new FileSystem instance is constructed. 336 * 337 * @param name a uri whose authority section names the host, port, etc. for this FileSystem 338 * @param conf the configuration 339 */ 340 @Override 341 public void initialize(URI name, Configuration conf) throws IOException { 342 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 343 344 //the real use is the one that has the Kerberos credentials needed for 345 //SPNEGO to work 346 realUser = ugi.getRealUser(); 347 if (realUser == null) { 348 realUser = UserGroupInformation.getLoginUser(); 349 } 350 super.initialize(name, conf); 351 try { 352 uri = new URI(name.getScheme() + "://" + name.getAuthority()); 353 } catch (URISyntaxException ex) { 354 throw new IOException(ex); 355 } 356 357 Class<? extends DelegationTokenAuthenticator> klass = 358 getConf().getClass("httpfs.authenticator.class", 359 KerberosDelegationTokenAuthenticator.class, 360 DelegationTokenAuthenticator.class); 361 DelegationTokenAuthenticator authenticator = 362 ReflectionUtils.newInstance(klass, getConf()); 363 authURL = new DelegationTokenAuthenticatedURL(authenticator); 364 } 365 366 @Override 367 public String getScheme() { 368 return SCHEME; 369 } 370 371 /** 372 * Returns a URI whose scheme and authority identify this FileSystem. 373 * 374 * @return the URI whose scheme and authority identify this FileSystem. 375 */ 376 @Override 377 public URI getUri() { 378 return uri; 379 } 380 381 /** 382 * Get the default port for this file system. 383 * @return the default port or 0 if there isn't one 384 */ 385 @Override 386 protected int getDefaultPort() { 387 return getConf().getInt(HdfsClientConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 388 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); 389 } 390 391 /** 392 * HttpFSServer subclass of the <code>FSDataInputStream</code>. 393 * <p> 394 * This implementation does not support the 395 * <code>PositionReadable</code> and <code>Seekable</code> methods. 396 */ 397 private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { 398 399 protected HttpFSDataInputStream(InputStream in, int bufferSize) { 400 super(new BufferedInputStream(in, bufferSize)); 401 } 402 403 @Override 404 public int read(long position, byte[] buffer, int offset, int length) throws IOException { 405 throw new UnsupportedOperationException(); 406 } 407 408 @Override 409 public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { 410 throw new UnsupportedOperationException(); 411 } 412 413 @Override 414 public void readFully(long position, byte[] buffer) throws IOException { 415 throw new UnsupportedOperationException(); 416 } 417 418 @Override 419 public void seek(long pos) throws IOException { 420 throw new UnsupportedOperationException(); 421 } 422 423 @Override 424 public long getPos() throws IOException { 425 throw new UnsupportedOperationException(); 426 } 427 428 @Override 429 public boolean seekToNewSource(long targetPos) throws IOException { 430 throw new UnsupportedOperationException(); 431 } 432 } 433 434 /** 435 * Opens an FSDataInputStream at the indicated Path. 436 * <p> 437 * IMPORTANT: the returned <code>FSDataInputStream</code> does not support the 438 * <code>PositionReadable</code> and <code>Seekable</code> methods. 439 * 440 * @param f the file name to open 441 * @param bufferSize the size of the buffer to be used. 442 */ 443 @Override 444 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 445 Map<String, String> params = new HashMap<String, String>(); 446 params.put(OP_PARAM, Operation.OPEN.toString()); 447 HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, 448 f, true); 449 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 450 return new FSDataInputStream( 451 new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); 452 } 453 454 /** 455 * HttpFSServer subclass of the <code>FSDataOutputStream</code>. 456 * <p> 457 * This implementation closes the underlying HTTP connection validating the Http connection status 458 * at closing time. 459 */ 460 private static class HttpFSDataOutputStream extends FSDataOutputStream { 461 private HttpURLConnection conn; 462 private int closeStatus; 463 464 public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) 465 throws IOException { 466 super(out, stats); 467 this.conn = conn; 468 this.closeStatus = closeStatus; 469 } 470 471 @Override 472 public void close() throws IOException { 473 try { 474 super.close(); 475 } finally { 476 HttpExceptionUtils.validateResponse(conn, closeStatus); 477 } 478 } 479 480 } 481 482 /** 483 * Converts a <code>FsPermission</code> to a Unix octal representation. 484 * 485 * @param p the permission. 486 * 487 * @return the Unix string symbolic reprentation. 488 */ 489 public static String permissionToString(FsPermission p) { 490 return Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8); 491 } 492 493 /* 494 * Common handling for uploading data for create and append operations. 495 */ 496 private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params, 497 int bufferSize, int expectedStatus) throws IOException { 498 HttpURLConnection conn = getConnection(method, params, f, true); 499 conn.setInstanceFollowRedirects(false); 500 boolean exceptionAlreadyHandled = false; 501 try { 502 if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) { 503 exceptionAlreadyHandled = true; 504 String location = conn.getHeaderField("Location"); 505 if (location != null) { 506 conn = getConnection(new URL(location), method); 507 conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE); 508 try { 509 OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); 510 return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); 511 } catch (IOException ex) { 512 HttpExceptionUtils.validateResponse(conn, expectedStatus); 513 throw ex; 514 } 515 } else { 516 HttpExceptionUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 517 throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); 518 } 519 } else { 520 throw new IOException( 521 MessageFormat.format("Expected HTTP status was [307], received [{0}]", 522 conn.getResponseCode())); 523 } 524 } catch (IOException ex) { 525 if (exceptionAlreadyHandled) { 526 throw ex; 527 } else { 528 HttpExceptionUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 529 throw ex; 530 } 531 } 532 } 533 534 535 /** 536 * Opens an FSDataOutputStream at the indicated Path with write-progress 537 * reporting. 538 * <p> 539 * IMPORTANT: The <code>Progressable</code> parameter is not used. 540 * 541 * @param f the file name to open. 542 * @param permission file permission. 543 * @param overwrite if a file with this name already exists, then if true, 544 * the file will be overwritten, and if false an error will be thrown. 545 * @param bufferSize the size of the buffer to be used. 546 * @param replication required block replication for the file. 547 * @param blockSize block size. 548 * @param progress progressable. 549 * 550 * @throws IOException 551 * @see #setPermission(Path, FsPermission) 552 */ 553 @Override 554 public FSDataOutputStream create(Path f, FsPermission permission, 555 boolean overwrite, int bufferSize, 556 short replication, long blockSize, 557 Progressable progress) throws IOException { 558 Map<String, String> params = new HashMap<String, String>(); 559 params.put(OP_PARAM, Operation.CREATE.toString()); 560 params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); 561 params.put(REPLICATION_PARAM, Short.toString(replication)); 562 params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); 563 params.put(PERMISSION_PARAM, permissionToString(permission)); 564 return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize, 565 HttpURLConnection.HTTP_CREATED); 566 } 567 568 569 /** 570 * Append to an existing file (optional operation). 571 * <p> 572 * IMPORTANT: The <code>Progressable</code> parameter is not used. 573 * 574 * @param f the existing file to be appended. 575 * @param bufferSize the size of the buffer to be used. 576 * @param progress for reporting progress if it is not null. 577 * 578 * @throws IOException 579 */ 580 @Override 581 public FSDataOutputStream append(Path f, int bufferSize, 582 Progressable progress) throws IOException { 583 Map<String, String> params = new HashMap<String, String>(); 584 params.put(OP_PARAM, Operation.APPEND.toString()); 585 return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize, 586 HttpURLConnection.HTTP_OK); 587 } 588 589 /** 590 * Truncate a file. 591 * 592 * @param f the file to be truncated. 593 * @param newLength The size the file is to be truncated to. 594 * 595 * @throws IOException 596 */ 597 @Override 598 public boolean truncate(Path f, long newLength) throws IOException { 599 Map<String, String> params = new HashMap<String, String>(); 600 params.put(OP_PARAM, Operation.TRUNCATE.toString()); 601 params.put(NEW_LENGTH_PARAM, Long.toString(newLength)); 602 HttpURLConnection conn = getConnection(Operation.TRUNCATE.getMethod(), 603 params, f, true); 604 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 605 return (Boolean) json.get(TRUNCATE_JSON); 606 } 607 608 /** 609 * Concat existing files together. 610 * @param f the path to the target destination. 611 * @param psrcs the paths to the sources to use for the concatenation. 612 * 613 * @throws IOException 614 */ 615 @Override 616 public void concat(Path f, Path[] psrcs) throws IOException { 617 List<String> strPaths = new ArrayList<String>(psrcs.length); 618 for(Path psrc : psrcs) { 619 strPaths.add(psrc.toUri().getPath()); 620 } 621 String srcs = StringUtils.join(",", strPaths); 622 623 Map<String, String> params = new HashMap<String, String>(); 624 params.put(OP_PARAM, Operation.CONCAT.toString()); 625 params.put(SOURCES_PARAM, srcs); 626 HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(), 627 params, f, true); 628 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 629 } 630 631 /** 632 * Renames Path src to Path dst. Can take place on local fs 633 * or remote DFS. 634 */ 635 @Override 636 public boolean rename(Path src, Path dst) throws IOException { 637 Map<String, String> params = new HashMap<String, String>(); 638 params.put(OP_PARAM, Operation.RENAME.toString()); 639 params.put(DESTINATION_PARAM, dst.toString()); 640 HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), 641 params, src, true); 642 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 643 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 644 return (Boolean) json.get(RENAME_JSON); 645 } 646 647 /** 648 * Delete a file. 649 * 650 * @deprecated Use delete(Path, boolean) instead 651 */ 652 @Deprecated 653 @Override 654 public boolean delete(Path f) throws IOException { 655 return delete(f, false); 656 } 657 658 /** 659 * Delete a file. 660 * 661 * @param f the path to delete. 662 * @param recursive if path is a directory and set to 663 * true, the directory is deleted else throws an exception. In 664 * case of a file the recursive can be set to either true or false. 665 * 666 * @return true if delete is successful else false. 667 * 668 * @throws IOException 669 */ 670 @Override 671 public boolean delete(Path f, boolean recursive) throws IOException { 672 Map<String, String> params = new HashMap<String, String>(); 673 params.put(OP_PARAM, Operation.DELETE.toString()); 674 params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); 675 HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), 676 params, f, true); 677 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 678 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 679 return (Boolean) json.get(DELETE_JSON); 680 } 681 682 /** 683 * List the statuses of the files/directories in the given path if the path is 684 * a directory. 685 * 686 * @param f given path 687 * 688 * @return the statuses of the files/directories in the given patch 689 * 690 * @throws IOException 691 */ 692 @Override 693 public FileStatus[] listStatus(Path f) throws IOException { 694 Map<String, String> params = new HashMap<String, String>(); 695 params.put(OP_PARAM, Operation.LISTSTATUS.toString()); 696 HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), 697 params, f, true); 698 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 699 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 700 json = (JSONObject) json.get(FILE_STATUSES_JSON); 701 JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); 702 FileStatus[] array = new FileStatus[jsonArray.size()]; 703 f = makeQualified(f); 704 for (int i = 0; i < jsonArray.size(); i++) { 705 array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i)); 706 } 707 return array; 708 } 709 710 /** 711 * Set the current working directory for the given file system. All relative 712 * paths will be resolved relative to it. 713 * 714 * @param newDir new directory. 715 */ 716 @Override 717 public void setWorkingDirectory(Path newDir) { 718 workingDir = newDir; 719 } 720 721 /** 722 * Get the current working directory for the given file system 723 * 724 * @return the directory pathname 725 */ 726 @Override 727 public Path getWorkingDirectory() { 728 if (workingDir == null) { 729 workingDir = getHomeDirectory(); 730 } 731 return workingDir; 732 } 733 734 /** 735 * Make the given file and all non-existent parents into 736 * directories. Has the semantics of Unix 'mkdir -p'. 737 * Existence of the directory hierarchy is not an error. 738 */ 739 @Override 740 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 741 Map<String, String> params = new HashMap<String, String>(); 742 params.put(OP_PARAM, Operation.MKDIRS.toString()); 743 params.put(PERMISSION_PARAM, permissionToString(permission)); 744 HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), 745 params, f, true); 746 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 747 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 748 return (Boolean) json.get(MKDIRS_JSON); 749 } 750 751 /** 752 * Return a file status object that represents the path. 753 * 754 * @param f The path we want information from 755 * 756 * @return a FileStatus object 757 * 758 * @throws FileNotFoundException when the path does not exist; 759 * IOException see specific implementation 760 */ 761 @Override 762 public FileStatus getFileStatus(Path f) throws IOException { 763 Map<String, String> params = new HashMap<String, String>(); 764 params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); 765 HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), 766 params, f, true); 767 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 768 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 769 json = (JSONObject) json.get(FILE_STATUS_JSON); 770 f = makeQualified(f); 771 return createFileStatus(f, json); 772 } 773 774 /** 775 * Return the current user's home directory in this filesystem. 776 * The default implementation returns "/user/$USER/". 777 */ 778 @Override 779 public Path getHomeDirectory() { 780 Map<String, String> params = new HashMap<String, String>(); 781 params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString()); 782 try { 783 HttpURLConnection conn = 784 getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, 785 new Path(getUri().toString(), "/"), false); 786 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 787 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 788 return new Path((String) json.get(HOME_DIR_JSON)); 789 } catch (IOException ex) { 790 throw new RuntimeException(ex); 791 } 792 } 793 794 /** 795 * Set owner of a path (i.e. a file or a directory). 796 * The parameters username and groupname cannot both be null. 797 * 798 * @param p The path 799 * @param username If it is null, the original username remains unchanged. 800 * @param groupname If it is null, the original groupname remains unchanged. 801 */ 802 @Override 803 public void setOwner(Path p, String username, String groupname) 804 throws IOException { 805 Map<String, String> params = new HashMap<String, String>(); 806 params.put(OP_PARAM, Operation.SETOWNER.toString()); 807 params.put(OWNER_PARAM, username); 808 params.put(GROUP_PARAM, groupname); 809 HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), 810 params, p, true); 811 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 812 } 813 814 /** 815 * Set permission of a path. 816 * 817 * @param p path. 818 * @param permission permission. 819 */ 820 @Override 821 public void setPermission(Path p, FsPermission permission) throws IOException { 822 Map<String, String> params = new HashMap<String, String>(); 823 params.put(OP_PARAM, Operation.SETPERMISSION.toString()); 824 params.put(PERMISSION_PARAM, permissionToString(permission)); 825 HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); 826 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 827 } 828 829 /** 830 * Set access time of a file 831 * 832 * @param p The path 833 * @param mtime Set the modification time of this file. 834 * The number of milliseconds since Jan 1, 1970. 835 * A value of -1 means that this call should not set modification time. 836 * @param atime Set the access time of this file. 837 * The number of milliseconds since Jan 1, 1970. 838 * A value of -1 means that this call should not set access time. 839 */ 840 @Override 841 public void setTimes(Path p, long mtime, long atime) throws IOException { 842 Map<String, String> params = new HashMap<String, String>(); 843 params.put(OP_PARAM, Operation.SETTIMES.toString()); 844 params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); 845 params.put(ACCESS_TIME_PARAM, Long.toString(atime)); 846 HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), 847 params, p, true); 848 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 849 } 850 851 /** 852 * Set replication for an existing file. 853 * 854 * @param src file name 855 * @param replication new replication 856 * 857 * @return true if successful; 858 * false if file does not exist or is a directory 859 * 860 * @throws IOException 861 */ 862 @Override 863 public boolean setReplication(Path src, short replication) 864 throws IOException { 865 Map<String, String> params = new HashMap<String, String>(); 866 params.put(OP_PARAM, Operation.SETREPLICATION.toString()); 867 params.put(REPLICATION_PARAM, Short.toString(replication)); 868 HttpURLConnection conn = 869 getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); 870 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 871 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 872 return (Boolean) json.get(SET_REPLICATION_JSON); 873 } 874 875 /** 876 * Modify the ACL entries for a file. 877 * 878 * @param path Path to modify 879 * @param aclSpec describing modifications 880 * @throws IOException 881 */ 882 @Override 883 public void modifyAclEntries(Path path, List<AclEntry> aclSpec) 884 throws IOException { 885 Map<String, String> params = new HashMap<String, String>(); 886 params.put(OP_PARAM, Operation.MODIFYACLENTRIES.toString()); 887 params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec)); 888 HttpURLConnection conn = getConnection( 889 Operation.MODIFYACLENTRIES.getMethod(), params, path, true); 890 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 891 } 892 893 /** 894 * Remove the specified ACL entries from a file 895 * @param path Path to modify 896 * @param aclSpec describing entries to remove 897 * @throws IOException 898 */ 899 @Override 900 public void removeAclEntries(Path path, List<AclEntry> aclSpec) 901 throws IOException { 902 Map<String, String> params = new HashMap<String, String>(); 903 params.put(OP_PARAM, Operation.REMOVEACLENTRIES.toString()); 904 params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec)); 905 HttpURLConnection conn = getConnection( 906 Operation.REMOVEACLENTRIES.getMethod(), params, path, true); 907 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 908 } 909 910 /** 911 * Removes the default ACL for the given file 912 * @param path Path from which to remove the default ACL. 913 * @throws IOException 914 */ 915 @Override 916 public void removeDefaultAcl(Path path) throws IOException { 917 Map<String, String> params = new HashMap<String, String>(); 918 params.put(OP_PARAM, Operation.REMOVEDEFAULTACL.toString()); 919 HttpURLConnection conn = getConnection( 920 Operation.REMOVEDEFAULTACL.getMethod(), params, path, true); 921 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 922 } 923 924 /** 925 * Remove all ACLs from a file 926 * @param path Path from which to remove all ACLs 927 * @throws IOException 928 */ 929 @Override 930 public void removeAcl(Path path) throws IOException { 931 Map<String, String> params = new HashMap<String, String>(); 932 params.put(OP_PARAM, Operation.REMOVEACL.toString()); 933 HttpURLConnection conn = getConnection(Operation.REMOVEACL.getMethod(), 934 params, path, true); 935 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 936 } 937 938 /** 939 * Set the ACLs for the given file 940 * @param path Path to modify 941 * @param aclSpec describing modifications, must include 942 * entries for user, group, and others for compatibility 943 * with permission bits. 944 * @throws IOException 945 */ 946 @Override 947 public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException { 948 Map<String, String> params = new HashMap<String, String>(); 949 params.put(OP_PARAM, Operation.SETACL.toString()); 950 params.put(ACLSPEC_PARAM, AclEntry.aclSpecToString(aclSpec)); 951 HttpURLConnection conn = getConnection(Operation.SETACL.getMethod(), 952 params, path, true); 953 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 954 } 955 956 /** 957 * Get the ACL information for a given file 958 * @param path Path to acquire ACL info for 959 * @return the ACL information in JSON format 960 * @throws IOException 961 */ 962 @Override 963 public AclStatus getAclStatus(Path path) throws IOException { 964 Map<String, String> params = new HashMap<String, String>(); 965 params.put(OP_PARAM, Operation.GETACLSTATUS.toString()); 966 HttpURLConnection conn = getConnection(Operation.GETACLSTATUS.getMethod(), 967 params, path, true); 968 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 969 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 970 json = (JSONObject) json.get(ACL_STATUS_JSON); 971 return createAclStatus(json); 972 } 973 974 /** Convert a string to a FsPermission object. */ 975 static FsPermission toFsPermission(JSONObject json) { 976 final String s = (String) json.get(PERMISSION_JSON); 977 final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON); 978 final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON); 979 FsPermission perm = new FsPermission(Short.parseShort(s, 8)); 980 final boolean aBit = (aclBit != null) ? aclBit : false; 981 final boolean eBit = (encBit != null) ? encBit : false; 982 if (aBit || eBit) { 983 return new FsPermissionExtension(perm, aBit, eBit); 984 } else { 985 return perm; 986 } 987 } 988 989 private FileStatus createFileStatus(Path parent, JSONObject json) { 990 String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); 991 Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); 992 FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); 993 long len = (Long) json.get(LENGTH_JSON); 994 String owner = (String) json.get(OWNER_JSON); 995 String group = (String) json.get(GROUP_JSON); 996 final FsPermission permission = toFsPermission(json); 997 long aTime = (Long) json.get(ACCESS_TIME_JSON); 998 long mTime = (Long) json.get(MODIFICATION_TIME_JSON); 999 long blockSize = (Long) json.get(BLOCK_SIZE_JSON); 1000 short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); 1001 FileStatus fileStatus = null; 1002 1003 switch (type) { 1004 case FILE: 1005 case DIRECTORY: 1006 fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), 1007 replication, blockSize, mTime, aTime, 1008 permission, owner, group, path); 1009 break; 1010 case SYMLINK: 1011 Path symLink = null; 1012 fileStatus = new FileStatus(len, false, 1013 replication, blockSize, mTime, aTime, 1014 permission, owner, group, symLink, 1015 path); 1016 } 1017 return fileStatus; 1018 } 1019 1020 /** 1021 * Convert the given JSON object into an AclStatus 1022 * @param json Input JSON representing the ACLs 1023 * @return Resulting AclStatus 1024 */ 1025 private AclStatus createAclStatus(JSONObject json) { 1026 AclStatus.Builder aclStatusBuilder = new AclStatus.Builder() 1027 .owner((String) json.get(OWNER_JSON)) 1028 .group((String) json.get(GROUP_JSON)) 1029 .stickyBit((Boolean) json.get(ACL_STICKY_BIT_JSON)); 1030 JSONArray entries = (JSONArray) json.get(ACL_ENTRIES_JSON); 1031 for ( Object e : entries ) { 1032 aclStatusBuilder.addEntry(AclEntry.parseAclEntry(e.toString(), true)); 1033 } 1034 return aclStatusBuilder.build(); 1035 } 1036 1037 @Override 1038 public ContentSummary getContentSummary(Path f) throws IOException { 1039 Map<String, String> params = new HashMap<String, String>(); 1040 params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); 1041 HttpURLConnection conn = 1042 getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); 1043 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1044 JSONObject json = (JSONObject) ((JSONObject) 1045 HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON); 1046 return new ContentSummary.Builder(). 1047 length((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON)). 1048 fileCount((Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON)). 1049 directoryCount((Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON)). 1050 quota((Long) json.get(CONTENT_SUMMARY_QUOTA_JSON)). 1051 spaceConsumed((Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON)). 1052 spaceQuota((Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)).build(); 1053 } 1054 1055 @Override 1056 public FileChecksum getFileChecksum(Path f) throws IOException { 1057 Map<String, String> params = new HashMap<String, String>(); 1058 params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); 1059 HttpURLConnection conn = 1060 getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); 1061 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1062 final JSONObject json = (JSONObject) ((JSONObject) 1063 HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON); 1064 return new FileChecksum() { 1065 @Override 1066 public String getAlgorithmName() { 1067 return (String) json.get(CHECKSUM_ALGORITHM_JSON); 1068 } 1069 1070 @Override 1071 public int getLength() { 1072 return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue(); 1073 } 1074 1075 @Override 1076 public byte[] getBytes() { 1077 return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON)); 1078 } 1079 1080 @Override 1081 public void write(DataOutput out) throws IOException { 1082 throw new UnsupportedOperationException(); 1083 } 1084 1085 @Override 1086 public void readFields(DataInput in) throws IOException { 1087 throw new UnsupportedOperationException(); 1088 } 1089 }; 1090 } 1091 1092 1093 @Override 1094 public Token<?> getDelegationToken(final String renewer) 1095 throws IOException { 1096 try { 1097 return UserGroupInformation.getCurrentUser().doAs( 1098 new PrivilegedExceptionAction<Token<?>>() { 1099 @Override 1100 public Token<?> run() throws Exception { 1101 return authURL.getDelegationToken(uri.toURL(), authToken, 1102 renewer); 1103 } 1104 } 1105 ); 1106 } catch (Exception ex) { 1107 if (ex instanceof IOException) { 1108 throw (IOException) ex; 1109 } else { 1110 throw new IOException(ex); 1111 } 1112 } 1113 } 1114 1115 public long renewDelegationToken(final Token<?> token) throws IOException { 1116 try { 1117 return UserGroupInformation.getCurrentUser().doAs( 1118 new PrivilegedExceptionAction<Long>() { 1119 @Override 1120 public Long run() throws Exception { 1121 return authURL.renewDelegationToken(uri.toURL(), authToken); 1122 } 1123 } 1124 ); 1125 } catch (Exception ex) { 1126 if (ex instanceof IOException) { 1127 throw (IOException) ex; 1128 } else { 1129 throw new IOException(ex); 1130 } 1131 } 1132 } 1133 1134 public void cancelDelegationToken(final Token<?> token) throws IOException { 1135 authURL.cancelDelegationToken(uri.toURL(), authToken); 1136 } 1137 1138 @Override 1139 public Token<?> getRenewToken() { 1140 return null; //TODO : for renewer 1141 } 1142 1143 @Override 1144 @SuppressWarnings("unchecked") 1145 public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) { 1146 //TODO : for renewer 1147 } 1148 1149 @Override 1150 public void setXAttr(Path f, String name, byte[] value, 1151 EnumSet<XAttrSetFlag> flag) throws IOException { 1152 Map<String, String> params = new HashMap<String, String>(); 1153 params.put(OP_PARAM, Operation.SETXATTR.toString()); 1154 params.put(XATTR_NAME_PARAM, name); 1155 if (value != null) { 1156 params.put(XATTR_VALUE_PARAM, 1157 XAttrCodec.encodeValue(value, XAttrCodec.HEX)); 1158 } 1159 params.put(XATTR_SET_FLAG_PARAM, EnumSetParam.toString(flag)); 1160 HttpURLConnection conn = getConnection(Operation.SETXATTR.getMethod(), 1161 params, f, true); 1162 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1163 } 1164 1165 @Override 1166 public byte[] getXAttr(Path f, String name) throws IOException { 1167 Map<String, String> params = new HashMap<String, String>(); 1168 params.put(OP_PARAM, Operation.GETXATTRS.toString()); 1169 params.put(XATTR_NAME_PARAM, name); 1170 HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(), 1171 params, f, true); 1172 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1173 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1174 Map<String, byte[]> xAttrs = createXAttrMap( 1175 (JSONArray) json.get(XATTRS_JSON)); 1176 return xAttrs != null ? xAttrs.get(name) : null; 1177 } 1178 1179 /** Convert xAttrs json to xAttrs map */ 1180 private Map<String, byte[]> createXAttrMap(JSONArray jsonArray) 1181 throws IOException { 1182 Map<String, byte[]> xAttrs = Maps.newHashMap(); 1183 for (Object obj : jsonArray) { 1184 JSONObject jsonObj = (JSONObject) obj; 1185 final String name = (String)jsonObj.get(XATTR_NAME_JSON); 1186 final byte[] value = XAttrCodec.decodeValue( 1187 (String)jsonObj.get(XATTR_VALUE_JSON)); 1188 xAttrs.put(name, value); 1189 } 1190 1191 return xAttrs; 1192 } 1193 1194 /** Convert xAttr names json to names list */ 1195 private List<String> createXAttrNames(String xattrNamesStr) throws IOException { 1196 JSONParser parser = new JSONParser(); 1197 JSONArray jsonArray; 1198 try { 1199 jsonArray = (JSONArray)parser.parse(xattrNamesStr); 1200 List<String> names = Lists.newArrayListWithCapacity(jsonArray.size()); 1201 for (Object name : jsonArray) { 1202 names.add((String) name); 1203 } 1204 return names; 1205 } catch (ParseException e) { 1206 throw new IOException("JSON parser error, " + e.getMessage(), e); 1207 } 1208 } 1209 1210 @Override 1211 public Map<String, byte[]> getXAttrs(Path f) throws IOException { 1212 Map<String, String> params = new HashMap<String, String>(); 1213 params.put(OP_PARAM, Operation.GETXATTRS.toString()); 1214 HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(), 1215 params, f, true); 1216 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1217 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1218 return createXAttrMap((JSONArray) json.get(XATTRS_JSON)); 1219 } 1220 1221 @Override 1222 public Map<String, byte[]> getXAttrs(Path f, List<String> names) 1223 throws IOException { 1224 Preconditions.checkArgument(names != null && !names.isEmpty(), 1225 "XAttr names cannot be null or empty."); 1226 Map<String, String> params = new HashMap<String, String>(); 1227 params.put(OP_PARAM, Operation.GETXATTRS.toString()); 1228 Map<String, List<String>> multiValuedParams = Maps.newHashMap(); 1229 multiValuedParams.put(XATTR_NAME_PARAM, names); 1230 HttpURLConnection conn = getConnection(Operation.GETXATTRS.getMethod(), 1231 params, multiValuedParams, f, true); 1232 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1233 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1234 return createXAttrMap((JSONArray) json.get(XATTRS_JSON)); 1235 } 1236 1237 @Override 1238 public List<String> listXAttrs(Path f) throws IOException { 1239 Map<String, String> params = new HashMap<String, String>(); 1240 params.put(OP_PARAM, Operation.LISTXATTRS.toString()); 1241 HttpURLConnection conn = getConnection(Operation.LISTXATTRS.getMethod(), 1242 params, f, true); 1243 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1244 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1245 return createXAttrNames((String) json.get(XATTRNAMES_JSON)); 1246 } 1247 1248 @Override 1249 public void removeXAttr(Path f, String name) throws IOException { 1250 Map<String, String> params = new HashMap<String, String>(); 1251 params.put(OP_PARAM, Operation.REMOVEXATTR.toString()); 1252 params.put(XATTR_NAME_PARAM, name); 1253 HttpURLConnection conn = getConnection(Operation.REMOVEXATTR.getMethod(), 1254 params, f, true); 1255 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1256 } 1257 1258 @Override 1259 public Collection<BlockStoragePolicy> getAllStoragePolicies() 1260 throws IOException { 1261 Map<String, String> params = new HashMap<String, String>(); 1262 params.put(OP_PARAM, Operation.GETALLSTORAGEPOLICY.toString()); 1263 HttpURLConnection conn = getConnection( 1264 Operation.GETALLSTORAGEPOLICY.getMethod(), params, new Path(getUri() 1265 .toString(), "/"), false); 1266 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1267 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1268 return createStoragePolicies((JSONObject) json.get(STORAGE_POLICIES_JSON)); 1269 } 1270 1271 private Collection<BlockStoragePolicy> createStoragePolicies(JSONObject map) 1272 throws IOException { 1273 JSONArray jsonArray = (JSONArray) map.get(STORAGE_POLICY_JSON); 1274 BlockStoragePolicy[] policies = new BlockStoragePolicy[jsonArray.size()]; 1275 for (int i = 0; i < jsonArray.size(); i++) { 1276 policies[i] = createStoragePolicy((JSONObject) jsonArray.get(i)); 1277 } 1278 return Arrays.asList(policies); 1279 } 1280 1281 @Override 1282 public BlockStoragePolicy getStoragePolicy(Path src) throws IOException { 1283 Map<String, String> params = new HashMap<String, String>(); 1284 params.put(OP_PARAM, Operation.GETSTORAGEPOLICY.toString()); 1285 HttpURLConnection conn = getConnection( 1286 Operation.GETSTORAGEPOLICY.getMethod(), params, src, true); 1287 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1288 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 1289 return createStoragePolicy((JSONObject) json.get(STORAGE_POLICY_JSON)); 1290 } 1291 private BlockStoragePolicy createStoragePolicy(JSONObject policyJson) 1292 throws IOException { 1293 byte id = ((Number) policyJson.get("id")).byteValue(); 1294 String name = (String) policyJson.get("name"); 1295 StorageType[] storageTypes = toStorageTypes((JSONArray) policyJson 1296 .get("storageTypes")); 1297 StorageType[] creationFallbacks = toStorageTypes((JSONArray) policyJson 1298 .get("creationFallbacks")); 1299 StorageType[] replicationFallbacks = toStorageTypes((JSONArray) policyJson 1300 .get("replicationFallbacks")); 1301 Boolean copyOnCreateFile = (Boolean) policyJson.get("copyOnCreateFile"); 1302 return new BlockStoragePolicy(id, name, storageTypes, creationFallbacks, 1303 replicationFallbacks, copyOnCreateFile.booleanValue()); 1304 } 1305 1306 private StorageType[] toStorageTypes(JSONArray array) throws IOException { 1307 if (array == null) { 1308 return null; 1309 } else { 1310 List<StorageType> storageTypes = new ArrayList<StorageType>(array.size()); 1311 for (Object name : array) { 1312 storageTypes.add(StorageType.parseStorageType((String) name)); 1313 } 1314 return storageTypes.toArray(new StorageType[storageTypes.size()]); 1315 } 1316 } 1317 1318 @Override 1319 public void setStoragePolicy(Path src, String policyName) throws IOException { 1320 Map<String, String> params = new HashMap<String, String>(); 1321 params.put(OP_PARAM, Operation.SETSTORAGEPOLICY.toString()); 1322 params.put(POLICY_NAME_PARAM, policyName); 1323 HttpURLConnection conn = getConnection( 1324 Operation.SETSTORAGEPOLICY.getMethod(), params, src, true); 1325 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1326 } 1327 1328 @Override 1329 public void unsetStoragePolicy(Path src) throws IOException { 1330 Map<String, String> params = new HashMap<String, String>(); 1331 params.put(OP_PARAM, Operation.UNSETSTORAGEPOLICY.toString()); 1332 HttpURLConnection conn = getConnection( 1333 Operation.UNSETSTORAGEPOLICY.getMethod(), params, src, true); 1334 HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 1335 } 1336}