# Copyright European Organization for Nuclear Research (CERN)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Ralph Vigne, <ralph.vigne@cern.ch>, 2012
# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2014
import hashlib
from exceptions import NotImplementedError
from urlparse import urlparse
from rucio.common import exception
from rucio.rse import rsemanager
if rsemanager.CLIENT_MODE:
from rucio.client.replicaclient import ReplicaClient
if rsemanager.SERVER_MODE:
from rucio.core import replica
[docs]class RSEProtocol(object):
""" This class is virtual and acts as a base to inherit new protocols from. It further provides some common functionality which applies for the amjority of the protocols."""
def __init__(self, protocol_attr, rse_settings):
""" Initializes the object with information about the referred RSE.
:param props: Properties of the reuested protocol
"""
self.attributes = protocol_attr
self.rse = rse_settings
if not self.rse['deterministic']:
if rsemanager.CLIENT_MODE:
setattr(self, 'lfns2pfns', self.__lfns2pfns_client)
if rsemanager.SERVER_MODE:
setattr(self, '_get_path', self._get_path_nondeterministic_server)
[docs] def lfns2pfns(self, lfns):
"""
Retruns a fully qualified PFN for the file referred by path.
:param path: The path to the file.
:returns: Fully qualified PFN.
"""
pfns = {}
prefix = self.attributes['prefix']
if not prefix.startswith('/'):
prefix = ''.join(['/', prefix])
if not prefix.endswith('/'):
prefix = ''.join([prefix, '/'])
lfns = [lfns] if type(lfns) == dict else lfns
for lfn in lfns:
scope, name = lfn['scope'], lfn['name']
pfns['%s:%s' % (scope, name)] = ''.join([self.attributes['scheme'], '://', self.attributes['hostname'], ':', str(self.attributes['port']), prefix, self._get_path(scope=scope, name=name)])
return pfns
def __lfns2pfns_client(self, lfns):
""" Provides the path of a replica for non-detemernisic sites. Will be assigned to get path by the __init__ method if neccessary.
:param scope: list of DIDs
:returns: dict with scope:name as keys and PFN as value (in case of errors the Rucio exception si assigned to the key)
"""
client = ReplicaClient()
pfns = {}
lfns = [lfns] if type(lfns) == dict else lfns
for lfn in lfns:
scope = lfn['scope']
name = lfn['name']
replicas = [r for r in client.list_replicas([{'scope': scope, 'name': name}, ], schemes=[self.attributes['scheme'], ])] # schemes is used to narrow down the response message.
if len(replicas) > 1:
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('This operation can only be performed for files.')
if not len(replicas):
pfns['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('File not found.')
pfns['%s:%s' % (scope, name)] = replicas[0]['rses'][self.rse['rse']][0] if (self.rse['rse'] in replicas[0]['rses'].keys()) else exception.RSEOperationNotSupported('Replica not found on given RSE.')
return pfns
def _get_path(self, scope, name):
""" Transforms the logical file name into a PFN.
Suitable for sites implementing the RUCIO naming convention.
:param lfn: filename
:param scope: scope
:returns: RSE specific URI of the physical file
"""
hstr = hashlib.md5('%s:%s' % (scope, name)).hexdigest()
correctedscope = "/".join(scope.split('.'))
return '%s/%s/%s/%s' % (correctedscope, hstr[0:2], hstr[2:4], name)
def _get_path_nondeterministic_server(self, scope, name):
""" Provides the path of a replica for non-detemernisic sites. Will be assigned to get path by the __init__ method if neccessary. """
r = replica.get_replica(rse=self.rse['rse'], scope=scope, name=name, rse_id=self.rse['id'])
if 'path' in r and r['path'] is not None:
path = r['path']
else:
raise exception.ReplicaNotFound('Missing path information for replica %s:%s on none-determinstic storage named %s' % (scope, name, self.rse['rse']))
if path.startswith('/'):
path = path[1:]
if path.endswith('/'):
path = path[:-1]
return path
[docs] def parse_pfns(self, pfns):
"""
Splits the given PFN into the parts known by the protocol. It is also checked if the provided protocol supportes the given PFNs.
:param pfns: a list of a fully qualified PFNs
:returns: dic with PFN as key and a dict with path and name as value
:raises RSEFileNameNotSupported: if the provided PFN doesn't match with the protocol settings
"""
ret = dict()
pfns = [pfns] if ((type(pfns) == str) or (type(pfns) == unicode)) else pfns
for pfn in pfns:
parsed = urlparse(pfn)
scheme = parsed.scheme
hostname = parsed.netloc.partition(':')[0]
port = int(parsed.netloc.partition(':')[2]) if parsed.netloc.partition(':')[2] != '' else 0
path = parsed.path
# Protect against 'lazy' defined prefixes for RSEs in the repository
if not self.attributes['prefix'].startswith('/'):
self.attributes['prefix'] = '/' + self.attributes['prefix']
if not self.attributes['prefix'].endswith('/'):
self.attributes['prefix'] += '/'
if self.attributes['hostname'] != hostname:
if self.attributes['hostname'] != 'localhost': # In the database empty hostnames are replaced with localhost but for some URIs (e.g. file) a hostname is not included
raise exception.RSEFileNameNotSupported('Invalid hostname: provided \'%s\', expected \'%s\'' % (hostname, self.attributes['hostname']))
if self.attributes['port'] != port:
raise exception.RSEFileNameNotSupported('Invalid port: provided \'%s\', expected \'%s\'' % (port, self.attributes['port']))
if not path.startswith(self.attributes['prefix']):
raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
self.attributes['prefix'])) # len(...)-1 due to the leading '/
# Spliting parsed.path into prefix, path, filename
prefix = self.attributes['prefix']
path = path.partition(self.attributes['prefix'])[2]
name = path.split('/')[-1]
path = path.partition(name)[0]
ret[pfn] = {'path': path, 'name': name, 'scheme': scheme, 'prefix': prefix, 'port': port, 'hostname': hostname, }
return ret
[docs] def exists(self, path):
"""
Checks if the requested file is known by the referred RSE.
:param path: Physical file name
:returns: True if the file exists, False if it doesn't
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
raise NotImplementedError
[docs] def connect(self):
"""
Establishes the actual connection to the referred RSE.
:raises RSEAccessDenied: if no connection could be established.
"""
raise NotImplementedError
[docs] def close(self):
""" Closes the connection to RSE."""
raise NotImplementedError
[docs] def get(self, path, dest):
"""
Provides access to files stored inside connected the RSE.
:param path: Physical file name of requested file
:param dest: Name and path of the files when stored at the client
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
raise NotImplementedError
[docs] def put(self, source, target, source_dir):
"""
Allows to store files inside the referred RSE.
:param source: path to the source file on the client file system
:param target: path to the destination file on the storage
:param source_dir: Path where the to be transferred files are stored in the local file system
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
raise NotImplementedError
[docs] def delete(self, path):
"""
Deletes a file from the connected RSE.
:param path: path to the to be deleted file
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
raise NotImplementedError
[docs] def rename(self, path, new_path):
""" Allows to rename a file stored inside the connected RSE.
:param path: path to the current file on the storage
:param new_path: path to the new file on the storage
:raises DestinationNotAccessible: if the destination storage was not accessible.
:raises ServiceUnavailable: if some generic error occured in the library.
:raises SourceNotFound: if the source file was not found on the referred storage.
"""
raise NotImplementedError