#!/usr/bin/env python

# reader-sru-retrieve.py - given an SRU query, cache content (and metadata) for the Reader

# Eric Lease Morgan <eric_morgan@infomotions.com>
# (c) Infomotions, LLC; distributed under a GNU Public License

# November 9, 2026 - written at least a year ago, but only documented today


# configure
SERVER         = 'http://catalog.distantreader.org:2100/biblios'
SCHEMA         = 'marcxml'
MAXIMUMRECORDS = 1024
COLUMNS        = [ 'author', 'title', 'date', 'url', 'file' ]
METADATA       = 'metadata.csv'
LABEL          = 'local/cached version'

# require
from   pathlib import Path
from   sruthi  import searchretrieve as search
import multiprocessing
import pandas as pd
import requests
import sys

# cache/download
def cache( directory, url ) :

	# parse
	filename = url[ 1 ]
	url      = url[ 0 ]
	
	# debug, request, and debug some more
	sys.stderr.write( 'Getting ' + url + ' ' )
	response = requests.get( url )
	code     = response.status_code
	sys.stderr.write( str( code ) + '\n' )
	
	# save, conditionally
	if code == 200 :
	
		with open( directory/filename, 'wb' ) as handle : handle.write( response.content )

# on our mark, get set, go
if __name__ == '__main__' : 

	# get input
	if len( sys.argv ) != 3 : sys.exit( "Usage: " + sys.argv[ 0 ] + " <query> <directory>" )
	query     = sys.argv[ 1 ]
	directory = sys.argv[ 2 ]

	# make sane
	directory = Path( directory )
	directory.mkdir( exist_ok=True )

	# initialize
	records = []

	# search and process each resulting record
	for index, record in enumerate( search( SERVER, maximum_records=MAXIMUMRECORDS, record_schema=SCHEMA, query=query ) ) :

		# re-initialize
		author = ''
		title  = ''
		date   = ''
		url    = ''
		file   = ''
		
		# process each field (tag)
		for field in record.get( 'datafield', [] ) :
		
			# get the current tag
			tag = field[ 'tag' ]
		
			# trap desired tags; author
			if tag   == '100' :
			
				try :
					if isinstance( field['subfield'], dict ) and field['subfield']['code'] == 'a' :
						author = field[ 'subfield' ][ 'text' ]
					elif isinstance( field['subfield'], list ) :
						for sub in field['subfield'] :
							if sub['code'] == 'a' : author = sub['text']
				except : pass
				
			# title
			elif tag == '245' :
			
				try :
					if isinstance( field['subfield'], dict ) and field['subfield']['code'] == 'a' :
						title = field[ 'subfield' ][ 'text' ]
					elif isinstance( field['subfield'], list ) :
						for sub in field['subfield'] :
							if sub['code'] == 'a' : title = sub['text']
				except : pass
		
			# date
			elif tag == '260' :
					
				# check for number of subfields; sometimes there are many
				if isinstance( field.get( 'subfield'), list ) : 
			
					# process all subfields
					for subfield in field.get( 'subfield' ) :
				
						# get the date, whew!
						try :
							if subfield[ 'code' ] == 'c' : date = subfield[ 'text' ]
						except : pass
						
				# check for single subfield, a dictionary
				elif isinstance( field.get( 'subfield' ), dict ) :
			
					# get the date; geesh!
					try : 
						if field.get( 'subfield' )[ 'code' ] == 'c' : date = field.get( 'subfield' )[ 'text' ]
					except : pass
					
				# error
				else : sys.exit( "Unknown type of instance for subfields of field 260; Call Eric.\n" )
			
			# added author
			elif tag == '700' and author == '' :
			
				try :
					if isinstance( field['subfield'], dict ) and field['subfield']['code'] == 'a' :
						author = field[ 'subfield' ][ 'text' ]
					elif isinstance( field['subfield'], list ) :
						for sub in field['subfield'] :
							if sub['code'] == 'a' :
								author = sub['text']
								break
				except : pass
					
			# url
			elif tag == '856' :
						
				# create a dictionary of the subfields; makes my life easier!
				subfields = {}
				for subfield in field[ 'subfield' ] : subfields[ subfield[ 'code' ] ] = subfield[ 'text' ]
				
				# look for the desired label
				if subfields[ 'z' ] == LABEL :
					
					# get the assoicated url and create a filename
					url  = subfields[ 'u' ]
					file = url.split( '/' )[ -1 ]			
		
		# sanity check; we only want downloadable items
		if ( url == '' ) : continue
			
		# debug
		sys.stderr.write( '      item: ' + str( index + 1 ) + '\n' )
		sys.stderr.write( '    author: ' + author + '\n' )
		sys.stderr.write( '     title: ' + title  + '\n' )
		sys.stderr.write( '      date: ' + date   + '\n' )
		sys.stderr.write( '       url: ' + url    + '\n' )
		sys.stderr.write( '      file: ' + file   + '\n' )
		sys.stderr.write( '\n' )
		
		# update
		record = [ author, title, date, url, file ]
		records.append( record )
	
	# create and output metdata
	metadata = pd.DataFrame( data=records, columns=COLUMNS )
	with open( directory/METADATA, 'w' ) as handle : handle.write( metadata.to_csv( index=False ) )
	
	# create a list of urls and file names, and then cache them; tricky
	urls = list( zip( metadata[ 'url' ].tolist(), metadata[ 'file' ].tolist() ) )	
	pool = multiprocessing.Pool()
	pool.starmap( cache, [ [ directory, url ] for url in urls ] )
	pool.close()

	# done
	exit()