diff -pruN 5.1-1/bin/get_character.py 6.6-1/bin/get_character.py --- 5.1-1/bin/get_character.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_character.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,12 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_character.py -Usage: get_character "characterID" +Usage: get_character "character_id" -Show some info about the character with the given characterID (e.g. '0000001' +Show some info about the character with the given character_id (e.g. '0000001' for "Jesse James", using 'http' or 'mobile'). -Notice that characterID, using 'sql', are not the same IDs used on the web. +Notice that character_id, using 'sql', are not the same IDs used on the web. """ import sys @@ -15,33 +16,32 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "characterID"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "character_id"' % sys.argv[0]) sys.exit(2) -characterID = sys.argv[1] +character_id = sys.argv[1] i = imdb.IMDb() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - try: # Get a character object with the data about the character identified by - # the given characterID. - character = i.get_character(characterID) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e + # the given character_id. + character = i.get_character(character_id) +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not character: - print 'It seems that there\'s no character with characterID "%s"' % characterID + print(("It seems that there's no character" + ' with character_id "%s"' % character_id)) sys.exit(4) # XXX: this is the easier way to print the main info about a character; @@ -51,6 +51,4 @@ if not character: # to access the data stored in a character object, so look below; the # commented lines show some ways to retrieve information from a # character object. -print character.summary().encode(out_encoding, 'replace') - - +print(character.summary()) diff -pruN 5.1-1/bin/get_company.py 6.6-1/bin/get_company.py --- 5.1-1/bin/get_company.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_company.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,12 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_company.py -Usage: get_company "companyID" +Usage: get_company "company_id" -Show some info about the company with the given companyID (e.g. '0071509' +Show some info about the company with the given company_id (e.g. '0071509' for "Columbia Pictures [us]", using 'http' or 'mobile'). -Notice that companyID, using 'sql', are not the same IDs used on the web. +Notice that company_id, using 'sql', are not the same IDs used on the web. """ import sys @@ -15,33 +16,31 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "companyID"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "company_id"' % sys.argv[0]) sys.exit(2) -companyID = sys.argv[1] +company_id = sys.argv[1] i = imdb.IMDb() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - try: # Get a company object with the data about the company identified by - # the given companyID. - company = i.get_company(companyID) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e + # the given company_id. + company = i.get_company(company_id) +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not company: - print 'It seems that there\'s no company with companyID "%s"' % companyID + print('It seems that there\'s no company with company_id "%s"' % company_id) sys.exit(4) # XXX: this is the easier way to print the main info about a company; @@ -51,6 +50,4 @@ if not company: # to access the data stored in a company object, so look below; the # commented lines show some ways to retrieve information from a # company object. -print company.summary().encode(out_encoding, 'replace') - - +print(company.summary()) diff -pruN 5.1-1/bin/get_first_character.py 6.6-1/bin/get_first_character.py --- 5.1-1/bin/get_first_character.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_first_character.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_first_character.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "character name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "character name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,24 +28,20 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of character objects). results = i.search_character(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not results: - print 'No matches for "%s", sorry.' % name.encode(out_encoding, 'replace') + print('No matches for "%s", sorry.' % name) sys.exit(0) # Print only the first result. -print ' Best match for "%s"' % name.encode(out_encoding, 'replace') +print(' Best match for "%s"' % name) # This is a character instance. character = results[0] @@ -53,7 +50,4 @@ character = results[0] # name; retrieve main information: i.update(character) -print character.summary().encode(out_encoding, 'replace') - - - +print(character.summary()) diff -pruN 5.1-1/bin/get_first_company.py 6.6-1/bin/get_first_company.py --- 5.1-1/bin/get_first_company.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_first_company.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_first_company.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "company name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "company name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,24 +28,20 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of company objects). results = i.search_company(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not results: - print 'No matches for "%s", sorry.' % name.encode(out_encoding, 'replace') + print('No matches for "%s", sorry.' % name) sys.exit(0) # Print only the first result. -print ' Best match for "%s"' % name.encode(out_encoding, 'replace') +print(' Best match for "%s"' % name) # This is a company instance. company = results[0] @@ -53,7 +50,4 @@ company = results[0] # name; retrieve main information: i.update(company) -print company.summary().encode(out_encoding, 'replace') - - - +print(company.summary()) diff -pruN 5.1-1/bin/get_first_movie.py 6.6-1/bin/get_first_movie.py --- 5.1-1/bin/get_first_movie.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_first_movie.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_first_movie.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "movie title"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "movie title"' % sys.argv[0]) sys.exit(2) title = sys.argv[1] @@ -27,24 +28,20 @@ title = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - -title = unicode(title, in_encoding, 'replace') try: # Do the search, and get the results (a list of Movie objects). results = i.search_movie(title) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not results: - print 'No matches for "%s", sorry.' % title.encode(out_encoding, 'replace') + print('No matches for "%s", sorry.' % title) sys.exit(0) # Print only the first result. -print ' Best match for "%s"' % title.encode(out_encoding, 'replace') +print(' Best match for "%s"' % title) # This is a Movie instance. movie = results[0] @@ -53,7 +50,4 @@ movie = results[0] # title and the year; retrieve main information: i.update(movie) -print movie.summary().encode(out_encoding, 'replace') - - - +print(movie.summary()) diff -pruN 5.1-1/bin/get_first_person.py 6.6-1/bin/get_first_person.py --- 5.1-1/bin/get_first_person.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_first_person.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_first_person.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "person name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "person name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,24 +28,20 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of Person objects). results = i.search_person(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not results: - print 'No matches for "%s", sorry.' % name.encode(out_encoding, 'replace') + print('No matches for "%s", sorry.' % name) sys.exit(0) # Print only the first result. -print ' Best match for "%s"' % name.encode(out_encoding, 'replace') +print(' Best match for "%s"' % name) # This is a Person instance. person = results[0] @@ -53,7 +50,4 @@ person = results[0] # name; retrieve main information: i.update(person) -print person.summary().encode(out_encoding, 'replace') - - - +print(person.summary()) diff -pruN 5.1-1/bin/get_keyword.py 6.6-1/bin/get_keyword.py --- 5.1-1/bin/get_keyword.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_keyword.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_keyword.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "keyword"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "keyword"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,27 +28,21 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of movies). results = i.get_keyword(name, results=20) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - name.encode(out_encoding, 'replace')) -print ' : movie title' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + name)) +print(' : movie title') # Print the long imdb title for every movie. for idx, movie in enumerate(results): - outp = u'%d: %s' % (idx+1, movie['long imdb title']) - print outp.encode(out_encoding, 'replace') - - + outp = '%d: %s' % (idx+1, movie['long imdb title']) + print(outp) diff -pruN 5.1-1/bin/get_movie.py 6.6-1/bin/get_movie.py --- 5.1-1/bin/get_movie.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_movie.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,12 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_movie.py -Usage: get_movie "movieID" +Usage: get_movie "movie_id" -Show some info about the movie with the given movieID (e.g. '0133093' +Show some info about the movie with the given movie_id (e.g. '0133093' for "The Matrix", using 'http' or 'mobile'). -Notice that movieID, using 'sql', are not the same IDs used on the web. +Notice that movie_id, using 'sql', are not the same IDs used on the web. """ import sys @@ -15,33 +16,31 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "movieID"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "movie_id"' % sys.argv[0]) sys.exit(2) -movieID = sys.argv[1] +movie_id = sys.argv[1] i = imdb.IMDb() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - try: # Get a Movie object with the data about the movie identified by - # the given movieID. - movie = i.get_movie(movieID) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e + # the given movie_id. + movie = i.get_movie(movie_id) +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not movie: - print 'It seems that there\'s no movie with movieID "%s"' % movieID + print('It seems that there\'s no movie with movie_id "%s"' % movie_id) sys.exit(4) # XXX: this is the easier way to print the main info about a movie; @@ -51,21 +50,22 @@ if not movie: # to access the data stored in a Movie object, so look below; the # commented lines show some ways to retrieve information from a # Movie object. -print movie.summary().encode(out_encoding, 'replace') +print(movie.summary()) # Show some info about the movie. # This is only a short example; you can get a longer summary using # 'print movie.summary()' and the complete set of information looking for # the output of the movie.keys() method. -#print '==== "%s" / movieID: %s ====' % (movie['title'], movieID) +# +# print '==== "%s" / movie_id: %s ====' % (movie['title'], movie_id) # XXX: use the IMDb instance to get the IMDb web URL for the movie. -#imdbURL = i.get_imdbURL(movie) -#if imdbURL: +# imdbURL = i.get_imdbURL(movie) +# if imdbURL: # print 'IMDb URL: %s' % imdbURL # # XXX: many keys return a list of values, like "genres". -#genres = movie.get('genres') -#if genres: +# genres = movie.get('genres') +# if genres: # print 'Genres: %s' % ' '.join(genres) # # XXX: even when only one value is present (e.g.: movie with only one @@ -73,8 +73,8 @@ print movie.summary().encode(out_encodin # Note that the 'name' variable is a Person object, but since its # __str__() method returns a string with the name, we can use it # directly, instead of name['name'] -#director = movie.get('director') -#if director: +# director = movie.get('director') +# if director: # print 'Director(s): ', # for name in director: # sys.stdout.write('%s ' % name) @@ -82,25 +82,23 @@ print movie.summary().encode(out_encodin # # XXX: notice that every name in the cast is a Person object, with a # currentRole instance variable, which is a string for the played role. -#cast = movie.get('cast') -#if cast: +# cast = movie.get('cast') +# if cast: # print 'Cast: ' # cast = cast[:5] # for name in cast: # print ' %s (%s)' % (name['name'], name.currentRole) # XXX: some information are not lists of strings or Person objects, but simple # strings, like 'rating'. -#rating = movie.get('rating') -#if rating: +# rating = movie.get('rating') +# if rating: # print 'Rating: %s' % rating # XXX: an example of how to use information sets; retrieve the "trivia" # info set; check if it contains some data, select and print a # random entry. -#import random -#i.update(movie, info=['trivia']) -#trivia = movie.get('trivia') -#if trivia: +# import random +# i.update(movie, info=['trivia']) +# trivia = movie.get('trivia') +# if trivia: # rand_trivia = trivia[random.randrange(len(trivia))] # print 'Random trivia: %s' % rand_trivia - - diff -pruN 5.1-1/bin/get_person.py 6.6-1/bin/get_person.py --- 5.1-1/bin/get_person.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_person.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,12 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_person.py -Usage: get_person "personID" +Usage: get_person "person_id" -Show some info about the person with the given personID (e.g. '0000210' +Show some info about the person with the given person_id (e.g. '0000210' for "Julia Roberts". -Notice that personID, using 'sql', are not the same IDs used on the web. +Notice that person_id, using 'sql', are not the same IDs used on the web. """ import sys @@ -15,33 +16,31 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "personID"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "person_id"' % sys.argv[0]) sys.exit(2) -personID = sys.argv[1] +person_id = sys.argv[1] i = imdb.IMDb() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - try: # Get a Person object with the data about the person identified by - # the given personID. - person = i.get_person(personID) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e + # the given person_id. + person = i.get_person(person_id) +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) if not person: - print 'It seems that there\'s no person with personID "%s"' % personID + print('It seems that there\'s no person with person_id "%s"' % person_id) sys.exit(4) # XXX: this is the easier way to print the main info about a person; @@ -51,40 +50,38 @@ if not person: # to access the data stored in a Person object, so look below; the # commented lines show some ways to retrieve information from a # Person object. -print person.summary().encode(out_encoding, 'replace') +print(person.summary()) # Show some info about the person. # This is only a short example; you can get a longer summary using # 'print person.summary()' and the complete set of information looking for # the output of the person.keys() method. -#print '==== "%s" / personID: %s ====' % (person['name'], personID) +# print '==== "%s" / person_id: %s ====' % (person['name'], person_id) # XXX: use the IMDb instance to get the IMDb web URL for the person. -#imdbURL = i.get_imdbURL(person) -#if imdbURL: +# imdbURL = i.get_imdbURL(person) +# if imdbURL: # print 'IMDb URL: %s' % imdbURL # XXX: print the birth date and birth notes. -#d_date = person.get('birth date') -#if d_date: +# d_date = person.get('birth date') +# if d_date: # print 'Birth date: %s' % d_date # b_notes = person.get('birth notes') # if b_notes: # print 'Birth notes: %s' % b_notes # XXX: print the last five movies he/she acted in, and the played role. -#movies_acted = person.get('actor') or person.get('actress') -#if movies_acted: +# movies_acted = person.get('actor') or person.get('actress') +# if movies_acted: # print 'Last roles played: ' # for movie in movies_acted[:5]: # print ' %s (in "%s")' % (movie.currentRole, movie['title']) # XXX: example of the use of information sets. -#import random -#i.update(person, info=['awards']) -#awards = person.get('awards') -#if awards: +# import random +# i.update(person, info=['awards']) +# awards = person.get('awards') +# if awards: # rand_award = awards[random.randrange(len(awards))] # s = 'Random award: in year ' # s += rand_award.get('year', '') # s += ' %s "%s"' % (rand_award.get('result', '').lower(), # rand_award.get('award', '')) # print s - - diff -pruN 5.1-1/bin/get_top_bottom_movies.py 6.6-1/bin/get_top_bottom_movies.py --- 5.1-1/bin/get_top_bottom_movies.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/get_top_bottom_movies.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ get_top_bottom_movies.py @@ -13,12 +14,12 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 1: - print 'No arguments are required.' + print('No arguments are required.') sys.exit(2) i = imdb.IMDb() @@ -26,14 +27,11 @@ i = imdb.IMDb() top250 = i.get_top250_movies() bottom100 = i.get_bottom100_movies() -out_encoding = sys.stdout.encoding or sys.getdefaultencoding() - for label, ml in [('top 10', top250[:10]), ('bottom 10', bottom100[:10])]: - print '' - print '%s movies' % label - print 'rating\tvotes\ttitle' + print('') + print('%s movies' % label) + print('rating\tvotes\ttitle') for movie in ml: - outl = u'%s\t%s\t%s' % (movie.get('rating'), movie.get('votes'), - movie['long imdb title']) - print outl.encode(out_encoding, 'replace') - + outl = '%s\t%s\t%s' % (movie.get('rating'), movie.get('votes'), + movie['long imdb title']) + print(outl) diff -pruN 5.1-1/bin/imdbpy2sql.py 6.6-1/bin/imdbpy2sql.py --- 5.1-1/bin/imdbpy2sql.py 2016-03-29 07:28:39.000000000 +0000 +++ 6.6-1/bin/imdbpy2sql.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,11 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ imdbpy2sql.py script. This script puts the data of the plain text data files into a SQL database. -Copyright 2005-2016 Davide Alberani +Copyright 2005-2017 Davide Alberani 2006 Giuseppe "Cowo" Corbelli lugbs.linux.it> This program is free software; you can redistribute it and/or modify @@ -29,27 +30,26 @@ import getopt import time import re import warnings -import anydbm +import operator +import dbm from itertools import islice, chain try: from hashlib import md5 except ImportError: from md5 import md5 from gzip import GzipFile -from types import UnicodeType -#from imdb.parser.sql.dbschema import * -from imdb.parser.sql.dbschema import DB_SCHEMA, dropTables, createTables, \ - createIndexes, createForeignKeys +from imdb.parser.sql.dbschema import DB_SCHEMA, dropTables, createTables, createIndexes from imdb.parser.sql import soundex from imdb.utils import analyze_title, analyze_name, date_and_notes, \ - build_name, build_title, normalizeName, normalizeTitle, _articles, \ + build_name, build_title, normalizeName, normalizeTitle, _articles, \ build_company_name, analyze_company_name, canonicalTitle from imdb._exceptions import IMDbParserError, IMDbError +from imdb.parser.sql.alchemyadapter import getDBTables, setConnection HELP = """imdbpy2sql.py usage: - %s -d /directory/with/PlainTextDataFiles/ -u URI [-c /directory/for/CSV_files] [-o sqlobject,sqlalchemy] [-i table,dbm] [--CSV-OPTIONS] [--COMPATIBILITY-OPTIONS] + %s -d /directory/with/PlainTextDataFiles/ -u URI [-c /directory/for/CSV_files] [-i table,dbm] [--CSV-OPTIONS] [--COMPATIBILITY-OPTIONS] # NOTE: URI is something along the line: scheme://[user[:password]@]host[:port]/database[?parameters] @@ -64,10 +64,6 @@ HELP = """imdbpy2sql.py usage: A directory is used to store CSV files; on supported database servers it should be really fast. - # NOTE: ORMs (-o orm): - Valid options are 'sqlobject', 'sqlalchemy' or the - preferred order separating the voices with a comma. - # NOTE: imdbIDs store/restore (-i method): Valid options are 'table' (imdbIDs stored in a temporary table of the database) or 'dbm' (imdbIDs stored on a dbm @@ -94,9 +90,6 @@ HELP = """imdbpy2sql.py usage: IMDB_PTDF_DIR = None # URI used to connect to the database. URI = None -# ORM to use (list of options) and actually used (string). -USE_ORM = None -USED_ORM = None # List of tables of the database. DB_TABLES = [] # Max allowed recursion, inserting data. @@ -121,45 +114,45 @@ CSV_PGSQL = "COPY %(table)s FROM '%(file CSV_DB2 = "CALL SYSPROC.ADMIN_CMD('LOAD FROM %(file)s OF del MODIFIED BY lobsinfile INSERT INTO %(table)s')" # Temporary fix for old style titles. -#FIX_OLD_STYLE_TITLES = True +# FIX_OLD_STYLE_TITLES = True # Store custom queries specified on the command line. CUSTOM_QUERIES = {} # Allowed time specification, for custom queries. ALLOWED_TIMES = ('BEGIN', 'BEFORE_DROP', 'BEFORE_CREATE', 'AFTER_CREATE', - 'BEFORE_MOVIES', 'BEFORE_COMPANIES', 'BEFORE_CAST', - 'BEFORE_RESTORE', 'BEFORE_INDEXES', 'END', 'BEFORE_MOVIES_TODB', - 'AFTER_MOVIES_TODB', 'BEFORE_PERSONS_TODB', - 'AFTER_PERSONS_TODB','BEFORE_SQLDATA_TODB', - 'AFTER_SQLDATA_TODB', 'BEFORE_AKAMOVIES_TODB', - 'AFTER_AKAMOVIES_TODB', 'BEFORE_CHARACTERS_TODB', - 'AFTER_CHARACTERS_TODB', 'BEFORE_COMPANIES_TODB', - 'AFTER_COMPANIES_TODB', 'BEFORE_EVERY_TODB', - 'AFTER_EVERY_TODB', 'BEFORE_CSV_LOAD', 'BEFORE_CSV_TODB', - 'AFTER_CSV_TODB') + 'BEFORE_MOVIES', 'BEFORE_COMPANIES', 'BEFORE_CAST', + 'BEFORE_RESTORE', 'BEFORE_INDEXES', 'END', 'BEFORE_MOVIES_TODB', + 'AFTER_MOVIES_TODB', 'BEFORE_PERSONS_TODB', + 'AFTER_PERSONS_TODB', 'BEFORE_SQLDATA_TODB', + 'AFTER_SQLDATA_TODB', 'BEFORE_AKAMOVIES_TODB', + 'AFTER_AKAMOVIES_TODB', 'BEFORE_CHARACTERS_TODB', + 'AFTER_CHARACTERS_TODB', 'BEFORE_COMPANIES_TODB', + 'AFTER_COMPANIES_TODB', 'BEFORE_EVERY_TODB', + 'AFTER_EVERY_TODB', 'BEFORE_CSV_LOAD', 'BEFORE_CSV_TODB', + 'AFTER_CSV_TODB') # Shortcuts for some compatibility options. MYSQLFORCEMYISAM_OPTS = ['-e', - 'AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;'] + 'AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;'] MYSQLINNODB_OPTS = ['-e', - 'AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;', - '-e', - 'BEFORE_INDEXES:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;'] -SQLSERVER_OPTS = ['-e', 'BEFORE_MOVIES_TODB:SET IDENTITY_INSERT %(table)s ON;', - '-e', 'AFTER_MOVIES_TODB:SET IDENTITY_INSERT %(table)s OFF;', - '-e', 'BEFORE_PERSONS_TODB:SET IDENTITY_INSERT %(table)s ON;', - '-e', 'AFTER_PERSONS_TODB:SET IDENTITY_INSERT %(table)s OFF;', - '-e', 'BEFORE_COMPANIES_TODB:SET IDENTITY_INSERT %(table)s ON;', - '-e', 'AFTER_COMPANIES_TODB:SET IDENTITY_INSERT %(table)s OFF;', - '-e', 'BEFORE_CHARACTERS_TODB:SET IDENTITY_INSERT %(table)s ON;', - '-e', 'AFTER_CHARACTERS_TODB:SET IDENTITY_INSERT %(table)s OFF;', - '-e', 'BEFORE_AKAMOVIES_TODB:SET IDENTITY_INSERT %(table)s ON;', - '-e', 'AFTER_AKAMOVIES_TODB:SET IDENTITY_INSERT %(table)s OFF;'] + 'AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;', + '-e', + 'BEFORE_INDEXES:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;'] +SQLSERVER_OPTS = ['-e', 'BEFORE_MOVIES_TODB:SET IDENTITY_INSERT %(table)s ON;', + '-e', 'AFTER_MOVIES_TODB:SET IDENTITY_INSERT %(table)s OFF;', + '-e', 'BEFORE_PERSONS_TODB:SET IDENTITY_INSERT %(table)s ON;', + '-e', 'AFTER_PERSONS_TODB:SET IDENTITY_INSERT %(table)s OFF;', + '-e', 'BEFORE_COMPANIES_TODB:SET IDENTITY_INSERT %(table)s ON;', + '-e', 'AFTER_COMPANIES_TODB:SET IDENTITY_INSERT %(table)s OFF;', + '-e', 'BEFORE_CHARACTERS_TODB:SET IDENTITY_INSERT %(table)s ON;', + '-e', 'AFTER_CHARACTERS_TODB:SET IDENTITY_INSERT %(table)s OFF;', + '-e', 'BEFORE_AKAMOVIES_TODB:SET IDENTITY_INSERT %(table)s ON;', + '-e', 'AFTER_AKAMOVIES_TODB:SET IDENTITY_INSERT %(table)s OFF;'] SQLITE_OPTS = ['-e', 'BEGIN:PRAGMA synchronous = OFF;', - '-e', 'BEFORE_EVERY_TODB:BEGIN TRANSACTION;', - '-e', 'AFTER_EVERY_TODB:COMMIT;', - '-e', 'BEFORE_INDEXES:BEGIN TRANSACTION;', - 'e', 'END:COMMIT;'] + '-e', 'BEFORE_EVERY_TODB:BEGIN TRANSACTION;', + '-e', 'AFTER_EVERY_TODB:COMMIT;', + '-e', 'BEFORE_INDEXES:BEGIN TRANSACTION;', + 'e', 'END:COMMIT;'] if '--mysql-innodb' in sys.argv[1:]: sys.argv += MYSQLINNODB_OPTS @@ -172,19 +165,19 @@ if '--sqlite-transactions' in sys.argv[1 # Manage arguments list. try: - optlist, args = getopt.getopt(sys.argv[1:], 'u:d:e:o:c:i:h', + optlist, args = getopt.getopt(sys.argv[1:], 'u:d:e:c:i:h', ['uri=', 'data=', 'execute=', - 'mysql-innodb', 'ms-sqlserver', - 'sqlite-transactions', - 'fix-old-style-titles', - 'mysql-force-myisam', 'orm', - 'csv-only-write', - 'csv-only-load', - 'csv=', 'csv-ext=', - 'imdbids=', 'help']) -except getopt.error, e: - print 'Troubles with arguments.' - print HELP + 'mysql-innodb', 'ms-sqlserver', + 'sqlite-transactions', + 'fix-old-style-titles', + 'mysql-force-myisam', + 'csv-only-write', + 'csv-only-load', + 'csv=', 'csv-ext=', + 'imdbids=', 'help']) +except getopt.error as e: + print('Troubles with arguments.') + print(HELP) sys.exit(2) for opt in optlist: @@ -200,26 +193,24 @@ for opt in optlist: IMDBIDS_METHOD = opt[1] elif opt[0] in ('-e', '--execute'): if opt[1].find(':') == -1: - print 'WARNING: wrong command syntax: "%s"' % opt[1] + print('WARNING: wrong command syntax: "%s"' % opt[1]) continue when, cmd = opt[1].split(':', 1) if when not in ALLOWED_TIMES: - print 'WARNING: unknown time: "%s"' % when + print('WARNING: unknown time: "%s"' % when) continue if when == 'BEFORE_EVERY_TODB': for nw in ('BEFORE_MOVIES_TODB', 'BEFORE_PERSONS_TODB', - 'BEFORE_SQLDATA_TODB', 'BEFORE_AKAMOVIES_TODB', - 'BEFORE_CHARACTERS_TODB', 'BEFORE_COMPANIES_TODB'): + 'BEFORE_SQLDATA_TODB', 'BEFORE_AKAMOVIES_TODB', + 'BEFORE_CHARACTERS_TODB', 'BEFORE_COMPANIES_TODB'): CUSTOM_QUERIES.setdefault(nw, []).append(cmd) elif when == 'AFTER_EVERY_TODB': for nw in ('AFTER_MOVIES_TODB', 'AFTER_PERSONS_TODB', - 'AFTER_SQLDATA_TODB', 'AFTER_AKAMOVIES_TODB', - 'AFTER_CHARACTERS_TODB', 'AFTER_COMPANIES_TODB'): + 'AFTER_SQLDATA_TODB', 'AFTER_AKAMOVIES_TODB', + 'AFTER_CHARACTERS_TODB', 'AFTER_COMPANIES_TODB'): CUSTOM_QUERIES.setdefault(nw, []).append(cmd) else: CUSTOM_QUERIES.setdefault(when, []).append(cmd) - elif opt[0] in ('-o', '--orm'): - USE_ORM = opt[1].split(',') elif opt[0] == '--fix-old-style-titles': warnings.warn('The --fix-old-style-titles argument is obsolete.') elif opt[0] == '--csv-only-write': @@ -227,27 +218,27 @@ for opt in optlist: elif opt[0] == '--csv-only-load': CSV_ONLY_LOAD = True elif opt[0] in ('-h', '--help'): - print HELP + print(HELP) sys.exit(0) if IMDB_PTDF_DIR is None: - print 'You must supply the directory with the plain text data files' - print HELP + print('You must supply the directory with the plain text data files') + print(HELP) sys.exit(2) if URI is None: - print 'You must supply the URI for the database connection' - print HELP + print('You must supply the URI for the database connection') + print(HELP) sys.exit(2) if IMDBIDS_METHOD not in (None, 'dbm', 'table'): - print 'the method to (re)store imdbIDs must be one of "dbm" or "table"' - print HELP + print('the method to (re)store imdbIDs must be one of "dbm" or "table"') + print(HELP) sys.exit(2) if (CSV_ONLY_WRITE or CSV_ONLY_LOAD) and not CSV_DIR: - print 'You must specify the CSV directory with the -c argument' - print HELP + print('You must specify the CSV directory with the -c argument') + print(HELP) sys.exit(3) @@ -257,44 +248,44 @@ URIlower = URI.lower() if URIlower.startswith('mysql'): if '--mysql-force-myisam' in sys.argv[1:] and \ '--mysql-innodb' in sys.argv[1:]: - print '\nWARNING: there is no sense in mixing the --mysql-innodb and\n'\ - '--mysql-force-myisam command line options!\n' + print('\nWARNING: there is no sense in mixing the --mysql-innodb and\n' + '--mysql-force-myisam command line options!\n') elif '--mysql-innodb' in sys.argv[1:]: - print "\nNOTICE: you've specified the --mysql-innodb command line\n"\ - "option; you should do this ONLY IF your system uses InnoDB\n"\ - "tables or you really want to use InnoDB; if you're running\n"\ - "a MyISAM-based database, please omit any option; if you\n"\ - "want to force MyISAM usage on a InnoDB-based database,\n"\ - "try the --mysql-force-myisam command line option, instead.\n" + print("\nNOTICE: you've specified the --mysql-innodb command line\n" + "option; you should do this ONLY IF your system uses InnoDB\n" + "tables or you really want to use InnoDB; if you're running\n" + "a MyISAM-based database, please omit any option; if you\n" + "want to force MyISAM usage on a InnoDB-based database,\n" + "try the --mysql-force-myisam command line option, instead.\n") elif '--mysql-force-myisam' in sys.argv[1:]: - print "\nNOTICE: you've specified the --mysql-force-myisam command\n"\ - "line option; you should do this ONLY IF your system uses\n"\ - "InnoDB tables and you want to use MyISAM tables, instead.\n" + print("\nNOTICE: you've specified the --mysql-force-myisam command\n" + "line option; you should do this ONLY IF your system uses\n" + "InnoDB tables and you want to use MyISAM tables, instead.\n") else: - print "\nNOTICE: IF you're using InnoDB tables, data insertion can\n"\ - "be very slow; you can switch to MyISAM tables - forcing it\n"\ - "with the --mysql-force-myisam option - OR use the\n"\ - "--mysql-innodb command line option, but DON'T USE these if\n"\ - "you're already working on MyISAM tables, because it will\n"\ - "force MySQL to use InnoDB, and performances will be poor.\n" + print("\nNOTICE: IF you're using InnoDB tables, data insertion can\n" + "be very slow; you can switch to MyISAM tables - forcing it\n" + "with the --mysql-force-myisam option - OR use the\n" + "--mysql-innodb command line option, but DON'T USE these if\n" + "you're already working on MyISAM tables, because it will\n" + "force MySQL to use InnoDB, and performances will be poor.\n") elif URIlower.startswith('mssql') and \ '--ms-sqlserver' not in sys.argv[1:]: - print "\nWARNING: you're using MS SQLServer without the --ms-sqlserver\n"\ - "command line option: if something goes wrong, try using it.\n" + print("\nWARNING: you're using MS SQLServer without the --ms-sqlserver\n" + "command line option: if something goes wrong, try using it.\n") elif URIlower.startswith('sqlite') and \ '--sqlite-transactions' not in sys.argv[1:]: - print "\nWARNING: you're using SQLite without the --sqlite-transactions\n"\ - "command line option: you'll have very poor performances! Try\n"\ - "using it.\n" + print("\nWARNING: you're using SQLite without the --sqlite-transactions\n" + "command line option: you'll have very poor performances! Try\n" + "using it.\n") if ('--mysql-force-myisam' in sys.argv[1:] and not URIlower.startswith('mysql')) or ('--mysql-innodb' in - sys.argv[1:] and not URIlower.startswith('mysql')) or ('--ms-sqlserver' - in sys.argv[1:] and not URIlower.startswith('mssql')) or \ + sys.argv[1:] and not URIlower.startswith('mysql')) or ('--ms-sqlserver' + in sys.argv[1:] and not URIlower.startswith('mssql')) or \ ('--sqlite-transactions' in sys.argv[1:] and - not URIlower.startswith('sqlite')): - print "\nWARNING: you've specified command line options that don't\n"\ - "belong to the database server you're using: proceed at your\n"\ - "own risk!\n" + not URIlower.startswith('sqlite')): + print("\nWARNING: you've specified command line options that don't\n" + "belong to the database server you're using: proceed at your\n" + "own risk!\n") if CSV_DIR: @@ -306,43 +297,14 @@ if CSV_DIR: CSV_LOAD_SQL = CSV_DB2 CSV_NULL = '' else: - print "\nERROR: importing CSV files is not supported for this database" - sys.exit(3) + print("\nERROR: importing CSV files is not supported for this database") + if not CSV_ONLY_WRITE: + sys.exit(3) -if USE_ORM is None: - USE_ORM = ('sqlobject', 'sqlalchemy') -if not isinstance(USE_ORM, (tuple, list)): - USE_ORM = [USE_ORM] -nrMods = len(USE_ORM) -_gotError = False -for idx, mod in enumerate(USE_ORM): - mod = mod.lower() - try: - if mod == 'sqlalchemy': - from imdb.parser.sql.alchemyadapter import getDBTables, setConnection - elif mod == 'sqlobject': - from imdb.parser.sql.objectadapter import getDBTables, setConnection - else: - warnings.warn('unknown module "%s".' % mod) - continue - DB_TABLES = getDBTables(URI) - for t in DB_TABLES: - globals()[t._imdbpyName] = t - if _gotError: - warnings.warn('falling back to "%s".' % mod) - USED_ORM = mod - break - except ImportError, e: - if idx+1 >= nrMods: - raise IMDbError('unable to use any ORM in %s: %s' % ( - str(USE_ORM), str(e))) - else: - warnings.warn('unable to use "%s": %s' % (mod, str(e))) - _gotError = True - continue -else: - raise IMDbError('unable to use any ORM in %s' % str(USE_ORM)) +DB_TABLES = getDBTables(URI) +for t in DB_TABLES: + globals()[t._imdbpyName] = t #----------------------- @@ -350,11 +312,13 @@ else: class CSVCursor(object): + """Emulate a cursor object, but instead it writes data to a set of CSV files.""" + def __init__(self, csvDir, csvExt=CSV_EXT, csvEOL=CSV_EOL, - delimeter=CSV_DELIMITER, quote=CSV_QUOTE, escape=CSV_ESCAPE, - null=CSV_NULL, quoteInteger=CSV_QUOTEINT): + delimeter=CSV_DELIMITER, quote=CSV_QUOTE, escape=CSV_ESCAPE, + null=CSV_NULL, quoteInteger=CSV_QUOTEINT): """Initialize a CSVCursor object; csvDir is the directory where the CSV files will be stored.""" self.csvDir = csvDir @@ -371,7 +335,7 @@ class CSVCursor(object): self._counters = {} def buildLine(self, items, tableToAddID=False, rawValues=(), - lobFD=None, lobFN=None): + lobFD=None, lobFN=None): """Build a single text line for a set of information.""" # FIXME: there are too many special cases to handle, and that # affects performances: management of LOB files, at least, @@ -391,7 +355,7 @@ class CSVCursor(object): if val is None: r[idx] = null continue - if (not quoteInteger) and isinstance(val, (int, long)): + if (not quoteInteger) and isinstance(val, int): r[idx] = str(val) continue if lobFD and idx == 3: @@ -419,7 +383,9 @@ class CSVCursor(object): r[3] = '%s.%d.%d/' % (lobFN, val3off, val3len) lobFD.write(val3) # Build the line and add the end-of-line. - return '%s%s' % (self.delimeter.join(r), self.csvEOL) + ret = '%s%s' % (self.delimeter.join(r), self.csvEOL) + ret = ret.encode('latin1', 'ignore') + return ret def executemany(self, sqlstr, items): """Emulate the executemany method of a cursor, but writes the @@ -449,8 +415,8 @@ class CSVCursor(object): buildLine = self.buildLine tableToAddID = False if tName in ('cast_info', 'movie_info', 'person_info', - 'movie_companies', 'movie_link', 'aka_name', - 'complete_cast', 'movie_info_idx', 'movie_keyword'): + 'movie_companies', 'movie_link', 'aka_name', + 'complete_cast', 'movie_info_idx', 'movie_keyword'): tableToAddID = tName if tName not in self._counters: self._counters[tName] = 1 @@ -458,29 +424,30 @@ class CSVCursor(object): # the query. parIdx = sqlstr.rfind('(') rawValues = [] - vals = sqlstr[parIdx+1:-1] + vals = sqlstr[parIdx + 1:-1] if parIdx != 0: - vals = sqlstr[parIdx+1:-1] + vals = sqlstr[parIdx + 1:-1] for idx, item in enumerate(vals.split(', ')): if item[0] in ('%', '?', ':'): continue rawValues.append((idx, item)) # Write these lines. tFD.writelines(buildLine(i, tableToAddID=tableToAddID, - rawValues=rawValues, lobFD=lobFD, lobFN=lobFN) - for i in items) + rawValues=rawValues, lobFD=lobFD, lobFN=lobFN) + for i in items) # Flush to disk, so that no truncaded entries are ever left. # XXX: is this a good idea? tFD.flush() def fileNames(self): """Return the list of file names.""" - return [fd.name for fd in self._fdPool.values()] + return [fd.name for fd in list(self._fdPool.values())] def buildFakeFileNames(self): """Populate the self._fdPool dictionary with fake objects taking file names from the content of the self.csvDir directory.""" - class _FakeFD(object): pass + class _FakeFD(object): + pass for fname in os.listdir(self.csvDir): if not fname.endswith(CSV_EXT): continue @@ -498,9 +465,9 @@ class CSVCursor(object): def closeAll(self): """Close all open file descriptors.""" - for fd in self._fdPool.values(): + for fd in list(self._fdPool.values()): fd.close() - for fd in self._lobFDPool.values(): + for fd in list(self._lobFDPool.values()): fd.close() @@ -515,7 +482,7 @@ def loadCSVFiles(): CSV_REPL['file'] = cfName CSV_REPL['table'] = tName sqlStr = CSV_LOAD_SQL % CSV_REPL - print ' * LOADING CSV FILE %s...' % cfName + print(' * LOADING CSV FILE %s...' % cfName) sys.stdout.flush() executeCustomQueries('BEFORE_CSV_TODB') try: @@ -523,11 +490,11 @@ def loadCSVFiles(): try: res = CURS.fetchall() if res: - print 'LOADING OUTPUT:', res + print('LOADING OUTPUT:', res) except: pass - except Exception, e: - print 'ERROR: unable to import CSV file %s: %s' % (cfName, str(e)) + except Exception as e: + print('ERROR: unable to import CSV file %s: %s' % (cfName, str(e))) continue connectObject.commit() executeCustomQueries('AFTER_CSV_TODB') @@ -543,23 +510,17 @@ if CSV_DIR: # Extract exceptions to trap. try: OperationalError = conn.module.OperationalError -except AttributeError, e: - warnings.warn('Unable to import OperationalError; report this as a bug, ' \ - 'since it will mask important exceptions: %s' % e) +except AttributeError as e: + warnings.warn('Unable to import OperationalError; report this as a bug, ' + 'since it will mask important exceptions: %s' % e) OperationalError = Exception try: IntegrityError = conn.module.IntegrityError -except AttributeError, e: +except AttributeError as e: warnings.warn('Unable to import IntegrityError') IntegrityError = Exception connectObject = conn.getConnection() -# XXX: fix for a problem that should be fixed in objectadapter.py (see it). -if URI and URI.startswith('sqlite') and USED_ORM == 'sqlobject': - major = sys.version_info[0] - minor = sys.version_info[1] - if major > 2 or (major == 2 and minor > 5): - connectObject.text_factory = str # Cursor object. CURS = connectObject.cursor() @@ -583,6 +544,7 @@ def tableName(table): """Return a string with the name of the table in the current db.""" return table.sqlmeta.table + def colName(table, column): """Return a string with the name of the column in the current db.""" if column == 'id': @@ -591,11 +553,14 @@ def colName(table, column): class RawValue(object): + """String-like objects to store raw SQL parameters, that are not intended to be replaced with positional parameters, in the query.""" + def __init__(self, s, v): self.string = s self.value = v + def __str__(self): return self.string @@ -605,15 +570,17 @@ def _makeConvNamed(cols): from positional style to named style (convert from a list of tuples to a list of dictionaries.""" nrCols = len(cols) + def _converter(params): for paramIndex, paramSet in enumerate(params): d = {} - for i in xrange(nrCols): + for i in range(nrCols): d[cols[i]] = paramSet[i] params[paramIndex] = d return params return _converter + def createSQLstr(table, cols, command='INSERT'): """Given a table and a list of columns returns a sql statement useful to insert a set of data in the database. @@ -624,13 +591,20 @@ def createSQLstr(table, cols, command='I values = [] convCols = [] count = 1 + def _valStr(s, index): - if DB_NAME in ('mysql', 'postgres'): return '%s' - elif PARAM_STYLE == 'format': return '%s' - elif PARAM_STYLE == 'qmark': return '?' - elif PARAM_STYLE == 'numeric': return ':%s' % index - elif PARAM_STYLE == 'named': return ':%s' % s - elif PARAM_STYLE == 'pyformat': return '%(' + s + ')s' + if DB_NAME in ('mysql', 'postgres'): + return '%s' + elif PARAM_STYLE == 'format': + return '%s' + elif PARAM_STYLE == 'qmark': + return '?' + elif PARAM_STYLE == 'numeric': + return ':%s' % index + elif PARAM_STYLE == 'named': + return ':%s' % s + elif PARAM_STYLE == 'pyformat': + return '%(' + s + ')s' return '%s' for col in cols: if isinstance(col, RawValue): @@ -656,20 +630,18 @@ def createSQLstr(table, cols, command='I converter = lambda x: x return sqlstr, converter + def _(s, truncateAt=None): """Nicely print a string to sys.stdout, optionally truncating it a the given char.""" - if not isinstance(s, UnicodeType): - s = unicode(s, 'utf_8') if truncateAt is not None: s = s[:truncateAt] - s = s.encode(sys.stdout.encoding or 'utf_8', 'replace') return s if not hasattr(os, 'times'): def times(): """Fake times() function.""" - return (0.0, 0.0, 0.0, 0.0, 0.0) + return 0.0, 0.0, 0.0, 0.0, 0.0 os.times = times # Show time consumed by the single function call. @@ -678,6 +650,7 @@ BEGIN_TIME = CTIME CTIMES = os.times() BEGIN_TIMES = CTIMES + def _minSec(*t): """Return a tuple of (mins, secs, ...) - two for every item passed.""" l = [] @@ -685,6 +658,7 @@ def _minSec(*t): l.extend(divmod(int(i), 60)) return tuple(l) + def t(s, sinceBegin=False): """Pretty-print timing information.""" global CTIME, CTIMES @@ -696,19 +670,21 @@ def t(s, sinceBegin=False): else: ct = BEGIN_TIME cts = BEGIN_TIMES - print '# TIME', s, \ - ': %dmin, %dsec (wall) %dmin, %dsec (user) %dmin, %dsec (system)' \ - % _minSec(nt-ct, ntimes[0]-cts[0], ntimes[1]-cts[1]) + print('# TIME', s, + ': %dmin, %dsec (wall) %dmin, %dsec (user) %dmin, %dsec (system)' + % _minSec(nt - ct, ntimes[0] - cts[0], ntimes[1] - cts[1])) if not sinceBegin: CTIME = nt CTIMES = ntimes + def title_soundex(title): """Return the soundex code for the given title; the (optional) starting article is pruned. It assumes to receive a title without year/imdbIndex or kind indications, but just the title string, as the one in the analyze_title(title)['title'] value.""" - if not title: return None + if not title: + return None # Convert to canonical format. title = canonicalTitle(title) ts = title.split(', ') @@ -717,6 +693,7 @@ def title_soundex(title): title = ', '.join(ts[:-1]) return soundex(title) + def name_soundexes(name, character=False): """Return three soundex codes for the given name; the name is assumed to be in the 'surname, name' format, without the imdbIndex indication, @@ -726,21 +703,21 @@ def name_soundexes(name, character=False from the first one. The third is the soundex of the surname, if different from the other two values.""" - ##if not isinstance(name, unicode): name = unicode(name, 'utf_8') - # Prune non-ascii chars from the string. - ##name = name.encode('ascii', 'ignore') - if not name: return (None, None, None) + if not name: + return None, None, None s1 = soundex(name) name_normal = normalizeName(name) s2 = soundex(name_normal) - if s1 == s2: s2 = None + if s1 == s2: + s2 = None if not character: namesplit = name.split(', ') s3 = soundex(namesplit[0]) else: s3 = soundex(name.split(' ')[-1]) - if s3 and s3 in (s1, s2): s3 = None - return (s1, s2, s3) + if s3 and s3 in (s1, s2): + s3 = None + return s1, s2, s3 # Tags to identify where the meaningful data begin/end in files. @@ -750,7 +727,7 @@ MOVIES_STOP = '------------------------- CAST_START = ('Name', '----') CAST_STOP = '-----------------------------' RAT_START = ('MOVIE RATINGS REPORT', '', - 'New Distribution Votes Rank Title') + 'New Distribution Votes Rank Title') RAT_STOP = '\n' RAT_TOP250_START = ('note: for this top 250', '', 'New Distribution') RAT_BOT10_START = ('BOTTOM 10 MOVIES', '', 'New Distribution') @@ -801,28 +778,35 @@ COMPCREW_START = ('CREW COVERAGE TRACKIN COMP_STOP = '---------------' GzipFileRL = GzipFile.readline + + class SourceFile(GzipFile): + """Instances of this class are used to read gzipped files, starting from a defined line to a (optionally) given end.""" + def __init__(self, filename=None, mode=None, start=(), stop=None, - pwarning=1, *args, **kwds): + pwarning=1, *args, **kwds): filename = os.path.join(IMDB_PTDF_DIR, filename) try: GzipFile.__init__(self, filename, mode, *args, **kwds) - except IOError, e: - if not pwarning: raise - print 'WARNING WARNING WARNING' - print 'WARNING unable to read the "%s" file.' % filename - print 'WARNING The file will be skipped, and the contained' - print 'WARNING information will NOT be stored in the database.' - print 'WARNING Complete error: ', e + except IOError as e: + if not pwarning: + raise + print('WARNING WARNING WARNING') + print('WARNING unable to read the "%s" file.' % filename) + print('WARNING The file will be skipped, and the contained') + print('WARNING information will NOT be stored in the database.') + print('WARNING Complete error: ', e) # re-raise the exception. raise self.start = start for item in start: itemlen = len(item) for line in self: - if line[:itemlen] == item: break + line = line.decode('latin1') + if line[:itemlen] == item: + break self.set_stop(stop) def set_stop(self, stop): @@ -835,12 +819,13 @@ class SourceFile(GzipFile): def readline_NOcheckEnd(self, size=-1): line = GzipFile.readline(self, size) - return unicode(line, 'latin_1').encode('utf_8') + return str(line, 'latin_1', 'ignore') def readline_checkEnd(self, size=-1): line = GzipFile.readline(self, size) - if self.stop is not None and line[:self.stoplen] == self.stop: return '' - return unicode(line, 'latin_1').encode('utf_8') + if self.stop is not None and line[:self.stoplen] == self.stop: + return '' + return str(line, 'latin_1', 'ignore') def getByHashSections(self): return getSectionHash(self) @@ -862,13 +847,16 @@ def getSectionHash(fp): curSectList[:] = [] curTitle = '' curTitle = line[2:] - else: curSectListApp(line) + else: + curSectListApp(line) if curSectList and curTitle: yield curTitle, joiner(curSectList) curSectList[:] = [] curTitle = '' NMMVSections = dict([(x, None) for x in ('MV: ', 'NM: ', 'OT: ', 'MOVI')]) + + def getSectionNMMV(fp): """Return sections separated by lines starting with 'NM: ', 'MV: ', 'OT: ' or 'MOVI'.""" @@ -882,14 +870,18 @@ def getSectionNMMV(fp): yield curNMMV, joiner(curSectList) curSectList[:] = [] curNMMV = '' - if line[:4] == 'MOVI': curNMMV = line[6:] - else: curNMMV = line[4:] - elif not (line and line[0] == '-'): curSectListApp(line) + if line[:4] == 'MOVI': + curNMMV = line[6:] + else: + curNMMV = line[4:] + elif not (line and line[0] == '-'): + curSectListApp(line) if curSectList and curNMMV: yield curNMMV, joiner(curSectList) curSectList[:] = [] curNMMV = '' + def counter(initValue=1): """A counter implemented using a generator.""" i = initValue @@ -897,8 +889,11 @@ def counter(initValue=1): yield i i += 1 + class _BaseCache(dict): + """Base class for Movie and Person basic information.""" + def __init__(self, d=None, flushEvery=100000): dict.__init__(self) # Flush data into the SQL database every flushEvery entries. @@ -910,7 +905,8 @@ class _BaseCache(dict): self._table_name = '' self._id_for_custom_q = '' if d is not None: - for k, v in d.iteritems(): self[k] = v + for k, v in d.items(): + self[k] = v def __setitem__(self, key, counter): """Every time a key is set, its value is the counter; @@ -926,11 +922,12 @@ class _BaseCache(dict): def flush(self, quiet=0, _recursionLevel=0): """Flush to the database.""" - if self._flushing: return + if self._flushing: + return self._flushing = 1 if _recursionLevel >= MAX_RECURSION: - print 'WARNING recursion level exceded trying to flush data' - print 'WARNING this batch of data is lost (%s).' % self.className + print('WARNING recursion level exceded trying to flush data') + print('WARNING this batch of data is lost (%s).' % self.className) self._tmpDict.clear() return if self._tmpDict: @@ -939,18 +936,18 @@ class _BaseCache(dict): keys = {'table': self._table_name} try: executeCustomQueries('BEFORE_%s_TODB' % self._id_for_custom_q, - _keys=keys, _timeit=False) + _keys=keys, _timeit=False) self._toDB(quiet) executeCustomQueries('AFTER_%s_TODB' % self._id_for_custom_q, - _keys=keys, _timeit=False) + _keys=keys, _timeit=False) _after_has_run = True self._tmpDict.clear() - except OperationalError, e: + except OperationalError as e: # XXX: I'm not sure this is the right thing (and way) # to proceed. if not _after_has_run: - executeCustomQueries('AFTER_%s_TODB'%self._id_for_custom_q, - _keys=keys, _timeit=False) + executeCustomQueries('AFTER_%s_TODB' % self._id_for_custom_q, + _keys=keys, _timeit=False) # Dataset too large; split it in two and retry. # XXX: new code! # the same class instance (self) is used, instead of @@ -960,31 +957,35 @@ class _BaseCache(dict): firstHalf = {} poptmpd = self._tmpDict.popitem originalLength = len(self._tmpDict) - for x in xrange(1, 1 + originalLength/2): + for x in range(1, 1 + originalLength // 2): k, v = poptmpd() firstHalf[k] = v self._secondHalf = self._tmpDict self._tmpDict = firstHalf - print ' * TOO MANY DATA (%s items in %s), recursion: %s' % \ - (originalLength, - self.className, - _recursionLevel) - print ' * SPLITTING (run 1 of 2), recursion: %s' % \ - _recursionLevel + print(' * TOO MANY DATA (%s items in %s), recursion: %s' % + (originalLength, + self.className, + _recursionLevel)) + print(' * SPLITTING (run 1 of 2), recursion: %s' % + _recursionLevel) self.flush(quiet=quiet, _recursionLevel=_recursionLevel) self._tmpDict = self._secondHalf - print ' * SPLITTING (run 2 of 2), recursion: %s' % \ - _recursionLevel + print(' * SPLITTING (run 2 of 2), recursion: %s' % + _recursionLevel) self.flush(quiet=quiet, _recursionLevel=_recursionLevel) self._tmpDict.clear() - except Exception, e: + except Exception as e: if isinstance(e, KeyboardInterrupt): raise - print 'WARNING: unknown exception caught committing the data' - print 'WARNING: to the database; report this as a bug, since' - print 'WARNING: many data (%d items) were lost: %s' % \ - (len(self._tmpDict), e) + print('WARNING: %s; unknown exception caught committing the data' % self.className) + print('WARNING: to the database; report this as a bug, since') + print('WARNING: many data (%d items) were lost: %s' % + (len(self._tmpDict), e)) self._flushing = 0 + try: + connectObject.commit() + except: + pass # Flush also deferred data. if self._deferredData: self._tmpDict = self._deferredData @@ -1002,9 +1003,7 @@ class _BaseCache(dict): def add(self, key, miscData=None): """Insert a new key and return its value.""" - c = self.counter.next() - # miscData=[('a_dict', 'value')] will set self.a_dict's c key - # to 'value'. + c = next(self.counter) if miscData is not None: for d_name, data in miscData: getattr(self, d_name)[c] = data @@ -1014,19 +1013,24 @@ class _BaseCache(dict): def addUnique(self, key, miscData=None): """Insert a new key and return its value; if the key is already in the dictionary, its previous value is returned.""" - if key in self: return self[key] - else: return self.add(key, miscData) + if key in self: + return self[key] + else: + return self.add(key, miscData) def fetchsome(curs, size=20000): """Yes, I've read the Python Cookbook! :-)""" while 1: res = curs.fetchmany(size) - if not res: break - for r in res: yield r + if not res: + break + for r in res: + yield r class MoviesCache(_BaseCache): + """Manage the movies list.""" className = 'MoviesCache' counter = counter() @@ -1037,13 +1041,13 @@ class MoviesCache(_BaseCache): self._table_name = tableName(Title) self._id_for_custom_q = 'MOVIES' self.sqlstr, self.converter = createSQLstr(Title, ('id', 'title', - 'imdbIndex', 'kindID', 'productionYear', - 'imdbID', 'phoneticCode', 'episodeOfID', - 'seasonNr', 'episodeNr', 'seriesYears', - 'md5sum')) + 'imdbIndex', 'kindID', 'productionYear', + 'imdbID', 'phoneticCode', 'episodeOfID', + 'seasonNr', 'episodeNr', 'seriesYears', + 'md5sum')) def populate(self): - print ' * POPULATING %s...' % self.className + print(' * POPULATING %s...' % self.className) titleTbl = tableName(Title) movieidCol = colName(Title, 'id') titleCol = colName(Title, 'title') @@ -1054,67 +1058,74 @@ class MoviesCache(_BaseCache): seasonNrCol = colName(Title, 'seasonNr') episodeNrCol = colName(Title, 'episodeNr') sqlPop = 'SELECT %s, %s, %s, %s, %s, %s, %s, %s FROM %s;' % \ - (movieidCol, titleCol, kindidCol, yearCol, imdbindexCol, + (movieidCol, titleCol, kindidCol, yearCol, imdbindexCol, episodeofidCol, seasonNrCol, episodeNrCol, titleTbl) CURS.execute(sqlPop) _oldcacheValues = Title.sqlmeta.cacheValues Title.sqlmeta.cacheValues = False for x in fetchsome(CURS, self.flushEvery): mdict = {'title': x[1], 'kind': KIND_STRS[x[2]], - 'year': x[3], 'imdbIndex': x[4]} - if mdict['imdbIndex'] is None: del mdict['imdbIndex'] - if mdict['year'] is None: del mdict['year'] - else: mdict['year'] = str(mdict['year']) + 'year': x[3], 'imdbIndex': x[4]} + if mdict['imdbIndex'] is None: + del mdict['imdbIndex'] + if mdict['year'] is None: + del mdict['year'] + else: + mdict['year'] = str(mdict['year']) episodeOfID = x[5] if episodeOfID is not None: s = Title.get(episodeOfID) series_d = {'title': s.title, 'kind': str(KIND_STRS[s.kindID]), 'year': s.productionYear, 'imdbIndex': s.imdbIndex} - if series_d['imdbIndex'] is None: del series_d['imdbIndex'] - if series_d['year'] is None: del series_d['year'] - else: series_d['year'] = str(series_d['year']) + if series_d['imdbIndex'] is None: + del series_d['imdbIndex'] + if series_d['year'] is None: + del series_d['year'] + else: + series_d['year'] = str(series_d['year']) mdict['episode of'] = series_d - title = build_title(mdict, ptdf=1, _emptyString='') + title = build_title(mdict, ptdf=True) dict.__setitem__(self, title, x[0]) self.counter = counter(Title.select().count() + 1) Title.sqlmeta.cacheValues = _oldcacheValues def _toDB(self, quiet=0): if not quiet: - print ' * FLUSHING %s...' % self.className + print(' * FLUSHING %s...' % self.className) sys.stdout.flush() l = [] lapp = l.append - tmpDictiter = self._tmpDict.iteritems - for k, v in tmpDictiter(): + for k, v in self._tmpDict.items(): try: - t = analyze_title(k, _emptyString='') + t = analyze_title(k) except IMDbParserError: if k and k.strip(): - print 'WARNING %s._toDB() invalid title:' % self.className, - print _(k) + print('WARNING %s._toDB() invalid title:' % self.className, end=' ') + print(_(k)) continue tget = t.get episodeOf = None kind = tget('kind') if kind == 'episode': # Series title. - stitle = build_title(tget('episode of'), _emptyString='', ptdf=1) + stitle = build_title(tget('episode of'), ptdf=True) episodeOf = self.addUnique(stitle) del t['episode of'] year = self.movieYear.get(v) if year is not None and year != '????': - try: t['year'] = int(year) - except ValueError: pass + try: + t['year'] = int(year) + except ValueError: + pass elif kind in ('tv series', 'tv mini series'): t['series years'] = self.movieYear.get(v) title = tget('title') soundex = title_soundex(title) lapp((v, title, tget('imdbIndex'), KIND_IDS[kind], - tget('year'), None, soundex, episodeOf, - tget('season'), tget('episode'), tget('series years'), - md5(k).hexdigest())) + tget('year'), None, soundex, episodeOf, + tget('season'), tget('episode'), tget('series years'), + md5(k.encode('latin1')).hexdigest())) self._runCommand(l) def _runCommand(self, dataList): @@ -1128,15 +1139,14 @@ class MoviesCache(_BaseCache): in the dictionary, its previous value is returned.""" if key.endswith('{{SUSPENDED}}'): return None - # DONE: to be removed when it will be no more needed! - #if FIX_OLD_STYLE_TITLES: - # key = build_title(analyze_title(key, canonical=False, - # _emptyString=''), ptdf=1, _emptyString='') - if key in self: return self[key] - else: return self.add(key, miscData) + if key in self: + return self[key] + else: + return self.add(key, miscData) class PersonsCache(_BaseCache): + """Manage the persons list.""" className = 'PersonsCache' counter = counter() @@ -1147,22 +1157,23 @@ class PersonsCache(_BaseCache): self._table_name = tableName(Name) self._id_for_custom_q = 'PERSONS' self.sqlstr, self.converter = createSQLstr(Name, ['id', 'name', - 'imdbIndex', 'imdbID', 'gender', 'namePcodeCf', - 'namePcodeNf', 'surnamePcode', 'md5sum']) + 'imdbIndex', 'imdbID', 'gender', 'namePcodeCf', + 'namePcodeNf', 'surnamePcode', 'md5sum']) def populate(self): - print ' * POPULATING PersonsCache...' + print(' * POPULATING PersonsCache...') nameTbl = tableName(Name) personidCol = colName(Name, 'id') nameCol = colName(Name, 'name') imdbindexCol = colName(Name, 'imdbIndex') CURS.execute('SELECT %s, %s, %s FROM %s;' % (personidCol, nameCol, - imdbindexCol, nameTbl)) + imdbindexCol, nameTbl)) _oldcacheValues = Name.sqlmeta.cacheValues Name.sqlmeta.cacheValues = False for x in fetchsome(CURS, self.flushEvery): nd = {'name': x[1]} - if x[2]: nd['imdbIndex'] = x[2] + if x[2]: + nd['imdbIndex'] = x[2] name = build_name(nd) dict.__setitem__(self, name, x[0]) self.counter = counter(Name.select().count() + 1) @@ -1170,25 +1181,24 @@ class PersonsCache(_BaseCache): def _toDB(self, quiet=0): if not quiet: - print ' * FLUSHING PersonsCache...' + print(' * FLUSHING PersonsCache...') sys.stdout.flush() l = [] lapp = l.append - tmpDictiter = self._tmpDict.iteritems - for k, v in tmpDictiter(): + for k, v in self._tmpDict.items(): try: t = analyze_name(k) except IMDbParserError: if k and k.strip(): - print 'WARNING PersonsCache._toDB() invalid name:', _(k) + print('WARNING PersonsCache._toDB() invalid name:', _(k)) continue tget = t.get name = tget('name') namePcodeCf, namePcodeNf, surnamePcode = name_soundexes(name) gender = self.personGender.get(v) lapp((v, name, tget('imdbIndex'), None, gender, - namePcodeCf, namePcodeNf, surnamePcode, - md5(k).hexdigest())) + namePcodeCf, namePcodeNf, surnamePcode, + md5(k.encode('latin1')).hexdigest())) if not CSV_DIR: CURS.executemany(self.sqlstr, self.converter(l)) else: @@ -1196,6 +1206,7 @@ class PersonsCache(_BaseCache): class CharactersCache(_BaseCache): + """Manage the characters list.""" counter = counter() className = 'CharactersCache' @@ -1205,22 +1216,23 @@ class CharactersCache(_BaseCache): self._table_name = tableName(CharName) self._id_for_custom_q = 'CHARACTERS' self.sqlstr, self.converter = createSQLstr(CharName, ['id', 'name', - 'imdbIndex', 'imdbID', 'namePcodeNf', - 'surnamePcode', 'md5sum']) + 'imdbIndex', 'imdbID', 'namePcodeNf', + 'surnamePcode', 'md5sum']) def populate(self): - print ' * POPULATING CharactersCache...' + print(' * POPULATING CharactersCache...') nameTbl = tableName(CharName) personidCol = colName(CharName, 'id') nameCol = colName(CharName, 'name') imdbindexCol = colName(CharName, 'imdbIndex') CURS.execute('SELECT %s, %s, %s FROM %s;' % (personidCol, nameCol, - imdbindexCol, nameTbl)) + imdbindexCol, nameTbl)) _oldcacheValues = CharName.sqlmeta.cacheValues CharName.sqlmeta.cacheValues = False for x in fetchsome(CURS, self.flushEvery): nd = {'name': x[1]} - if x[2]: nd['imdbIndex'] = x[2] + if x[2]: + nd['imdbIndex'] = x[2] name = build_name(nd) dict.__setitem__(self, name, x[0]) self.counter = counter(CharName.select().count() + 1) @@ -1228,24 +1240,23 @@ class CharactersCache(_BaseCache): def _toDB(self, quiet=0): if not quiet: - print ' * FLUSHING CharactersCache...' + print(' * FLUSHING CharactersCache...') sys.stdout.flush() l = [] lapp = l.append - tmpDictiter = self._tmpDict.iteritems - for k, v in tmpDictiter(): + for k, v in self._tmpDict.items(): try: t = analyze_name(k) except IMDbParserError: if k and k.strip(): - print 'WARNING CharactersCache._toDB() invalid name:', _(k) + print('WARNING CharactersCache._toDB() invalid name:', _(k)) continue tget = t.get name = tget('name') namePcodeCf, namePcodeNf, surnamePcode = name_soundexes(name, - character=True) + character=True) lapp((v, name, tget('imdbIndex'), None, - namePcodeCf, surnamePcode, md5(k).hexdigest())) + namePcodeCf, surnamePcode, md5(k.encode('latin1')).hexdigest())) if not CSV_DIR: CURS.executemany(self.sqlstr, self.converter(l)) else: @@ -1253,6 +1264,7 @@ class CharactersCache(_BaseCache): class CompaniesCache(_BaseCache): + """Manage the companies list.""" counter = counter() className = 'CompaniesCache' @@ -1262,22 +1274,23 @@ class CompaniesCache(_BaseCache): self._table_name = tableName(CompanyName) self._id_for_custom_q = 'COMPANIES' self.sqlstr, self.converter = createSQLstr(CompanyName, ['id', 'name', - 'countryCode', 'imdbID', 'namePcodeNf', - 'namePcodeSf', 'md5sum']) + 'countryCode', 'imdbID', 'namePcodeNf', + 'namePcodeSf', 'md5sum']) def populate(self): - print ' * POPULATING CharactersCache...' + print(' * POPULATING CharactersCache...') nameTbl = tableName(CompanyName) companyidCol = colName(CompanyName, 'id') nameCol = colName(CompanyName, 'name') countryCodeCol = colName(CompanyName, 'countryCode') CURS.execute('SELECT %s, %s, %s FROM %s;' % (companyidCol, nameCol, - countryCodeCol, nameTbl)) + countryCodeCol, nameTbl)) _oldcacheValues = CompanyName.sqlmeta.cacheValues CompanyName.sqlmeta.cacheValues = False for x in fetchsome(CURS, self.flushEvery): nd = {'name': x[1]} - if x[2]: nd['country'] = x[2] + if x[2]: + nd['country'] = x[2] name = build_company_name(nd) dict.__setitem__(self, name, x[0]) self.counter = counter(CompanyName.select().count() + 1) @@ -1285,17 +1298,16 @@ class CompaniesCache(_BaseCache): def _toDB(self, quiet=0): if not quiet: - print ' * FLUSHING CompaniesCache...' + print(' * FLUSHING CompaniesCache...') sys.stdout.flush() l = [] lapp = l.append - tmpDictiter = self._tmpDict.iteritems - for k, v in tmpDictiter(): + for k, v in self._tmpDict.items(): try: t = analyze_company_name(k) except IMDbParserError: if k and k.strip(): - print 'WARNING CompaniesCache._toDB() invalid name:', _(k) + print('WARNING CompaniesCache._toDB() invalid name:', _(k)) continue tget = t.get name = tget('name') @@ -1305,7 +1317,7 @@ class CompaniesCache(_BaseCache): if k != name: namePcodeSf = soundex(k) lapp((v, name, country, None, namePcodeNf, namePcodeSf, - md5(k).hexdigest())) + md5(k.encode('latin1')).hexdigest())) if not CSV_DIR: CURS.executemany(self.sqlstr, self.converter(l)) else: @@ -1313,6 +1325,7 @@ class CompaniesCache(_BaseCache): class KeywordsCache(_BaseCache): + """Manage the list of keywords.""" counter = counter() className = 'KeywordsCache' @@ -1323,15 +1336,15 @@ class KeywordsCache(_BaseCache): self._id_for_custom_q = 'KEYWORDS' self.flushEvery = 10000 self.sqlstr, self.converter = createSQLstr(Keyword, ['id', 'keyword', - 'phoneticCode']) + 'phoneticCode']) def populate(self): - print ' * POPULATING KeywordsCache...' + print(' * POPULATING KeywordsCache...') nameTbl = tableName(CompanyName) keywordidCol = colName(Keyword, 'id') keyCol = colName(Keyword, 'name') CURS.execute('SELECT %s, %s FROM %s;' % (keywordidCol, keyCol, - nameTbl)) + nameTbl)) _oldcacheValues = Keyword.sqlmeta.cacheValues Keyword.sqlmeta.cacheValues = False for x in fetchsome(CURS, self.flushEvery): @@ -1341,12 +1354,11 @@ class KeywordsCache(_BaseCache): def _toDB(self, quiet=0): if not quiet: - print ' * FLUSHING KeywordsCache...' + print(' * FLUSHING KeywordsCache...') sys.stdout.flush() l = [] lapp = l.append - tmpDictiter = self._tmpDict.iteritems - for k, v in tmpDictiter(): + for k, v in self._tmpDict.items(): keySoundex = soundex(k) lapp((v, k, keySoundex)) if not CSV_DIR: @@ -1356,10 +1368,12 @@ class KeywordsCache(_BaseCache): class SQLData(dict): + """Variable set of information, to be stored from time to time to the SQL database.""" + def __init__(self, table=None, cols=None, sqlString='', converter=None, - d={}, flushEvery=20000, counterInit=1): + d={}, flushEvery=20000, counterInit=1): if not sqlString: if not (table and cols): raise TypeError('"table" or "cols" unspecified') @@ -1375,7 +1389,8 @@ class SQLData(dict): self._recursionLevel = 1 self._table = table self._table_name = tableName(table) - for k, v in d.items(): self[k] = v + for k, v in list(d.items()): + self[k] = v def __setitem__(self, key, value): """The value is discarded, the counter is used as the 'real' key @@ -1390,16 +1405,16 @@ class SQLData(dict): self[key] = None def flush(self, _resetRecursion=1): - if not self: return - # XXX: it's safer to flush MoviesCache and PersonsCache, to preserve - # consistency of ForeignKey, but it can also slow down everything - # a bit... + if not self: + return + # XXX: it's safer to flush MoviesCache and PersonsCache, to preserve consistency CACHE_MID.flush(quiet=1) CACHE_PID.flush(quiet=1) - if _resetRecursion: self._recursionLevel = 1 + if _resetRecursion: + self._recursionLevel = 1 if self._recursionLevel >= MAX_RECURSION: - print 'WARNING recursion level exceded trying to flush data' - print 'WARNING this batch of data is lost.' + print('WARNING recursion level exceded trying to flush data') + print('WARNING this batch of data is lost.') self.clear() self.counter = self.counterInit return @@ -1407,28 +1422,28 @@ class SQLData(dict): _after_has_run = False try: executeCustomQueries('BEFORE_SQLDATA_TODB', _keys=keys, - _timeit=False) + _timeit=False) self._toDB() executeCustomQueries('AFTER_SQLDATA_TODB', _keys=keys, - _timeit=False) + _timeit=False) _after_has_run = True self.clear() self.counter = self.counterInit - except OperationalError, e: + except OperationalError as e: if not _after_has_run: executeCustomQueries('AFTER_SQLDATA_TODB', _keys=keys, - _timeit=False) - print ' * TOO MANY DATA (%s items), SPLITTING (run #%d)...' % \ - (len(self), self._recursionLevel) + _timeit=False) + print(' * TOO MANY DATA (%s items), SPLITTING (run #%d)...' % + (len(self), self._recursionLevel)) self._recursionLevel += 1 newdata = self.__class__(table=self._table, - sqlString=self.sqlString, - converter=self.converter) + sqlString=self.sqlString, + converter=self.converter) newdata._recursionLevel = self._recursionLevel - newflushEvery = self.flushEvery / 2 + newflushEvery = self.flushEvery // 2 if newflushEvery < 1: - print 'WARNING recursion level exceded trying to flush data' - print 'WARNING this batch of data is lost.' + print('WARNING recursion level exceded trying to flush data') + print('WARNING this batch of data is lost.') self.clear() self.counter = self.counterInit return @@ -1436,7 +1451,7 @@ class SQLData(dict): newdata.flushEvery = newflushEvery popitem = self.popitem dsi = dict.__setitem__ - for x in xrange(len(self)/2): + for x in range(len(self) // 2): k, v = popitem() dsi(newdata, k, v) newdata.flush(_resetRecursion=0) @@ -1444,21 +1459,21 @@ class SQLData(dict): self.flush(_resetRecursion=0) self.clear() self.counter = self.counterInit - except Exception, e: + except Exception as e: if isinstance(e, KeyboardInterrupt): raise - print 'WARNING: unknown exception caught committing the data' - print 'WARNING: to the database; report this as a bug, since' - print 'WARNING: many data (%d items) were lost: %s' % \ - (len(self), e) + print('WARNING: SQLData; unknown exception caught committing the data') + print('WARNING: to the database; report this as a bug, since') + print('WARNING: many data (%d items) were lost: %s' % + (len(self), e)) connectObject.commit() def _toDB(self): - print ' * FLUSHING SQLData...' + print(' * FLUSHING SQLData...') if not CSV_DIR: - CURS.executemany(self.sqlString, self.converter(self.values())) + CURS.executemany(self.sqlString, self.converter(list(self.values()))) else: - CSV_CURS.executemany(self.sqlString, self.values()) + CSV_CURS.executemany(self.sqlString, list(self.values())) # Miscellaneous functions. @@ -1475,27 +1490,35 @@ def unpack(line, headers, sep='\t'): 'rating': '8.4', 'title': 'Incredibles, The (2004)'} """ r = {} - ls1 = filter(None, line.split(sep)) + ls1 = [_f for _f in line.split(sep) if _f] for index, item in enumerate(ls1): - try: name = headers[index] - except IndexError: name = 'item%s' % index + try: + name = headers[index] + except IndexError: + name = 'item%s' % index r[name] = item.strip() return r + def _parseMinusList(fdata): """Parse a list of lines starting with '- '.""" rlist = [] tmplist = [] for line in fdata: if line and line[:2] == '- ': - if tmplist: rlist.append(' '.join(tmplist)) + if tmplist: + rlist.append(' '.join(tmplist)) l = line[2:].strip() - if l: tmplist[:] = [l] - else: tmplist[:] = [] + if l: + tmplist[:] = [l] + else: + tmplist[:] = [] else: l = line.strip() - if l: tmplist.append(l) - if tmplist: rlist.append(' '.join(tmplist)) + if l: + tmplist.append(l) + if tmplist: + rlist.append(' '.join(tmplist)) return rlist @@ -1504,13 +1527,16 @@ def _parseColonList(lines, replaceKeys): out = {} for line in lines: line = line.strip() - if not line: continue + if not line: + continue cols = line.split(':', 1) - if len(cols) < 2: continue + if len(cols) < 2: + continue k = cols[0] k = replaceKeys.get(k, k) v = ' '.join(cols[1:]).strip() - if k not in out: out[k] = [] + if k not in out: + out[k] = [] out[k].append(v) return out @@ -1519,8 +1545,10 @@ def _parseColonList(lines, replaceKeys): def readMovieList(): """Read the movies.list.gz file.""" - try: mdbf = SourceFile(MOVIES, start=MOVIES_START, stop=MOVIES_STOP) - except IOError: return + try: + mdbf = SourceFile(MOVIES, start=MOVIES_START, stop=MOVIES_STOP) + except IOError: + return count = 0 for line in mdbf: line_d = unpack(line, ('title', 'year')) @@ -1533,8 +1561,8 @@ def readMovieList(): if mid is None: continue if count % 10000 == 0: - print 'SCANNING movies:', _(title), - print '(movieID: %s)' % mid + print('SCANNING movies:', _(title), end=' ') + print('(movieID: %s)' % mid) count += 1 CACHE_MID.flush() CACHE_MID.movieYear.clear() @@ -1548,13 +1576,16 @@ def doCast(fp, roleid, rolename): name = '' roleidVal = RawValue('roleID', roleid) sqldata = SQLData(table=CastInfo, cols=['personID', 'movieID', - 'personRoleID', 'note', 'nrOrder', roleidVal]) - if rolename == 'miscellaneous crew': sqldata.flushEvery = 10000 + 'personRoleID', 'note', 'nrOrder', roleidVal]) + if rolename == 'miscellaneous crew': + sqldata.flushEvery = 10000 for line in fp: if line and line[0] != '\t': - if line[0] == '\n': continue - sl = filter(None, line.split('\t')) - if len(sl) != 2: continue + if line[0] == '\n': + continue + sl = [_f for _f in line.split('\t') if _f] + if len(sl) != 2: + continue name, line = sl miscData = None if rolename == 'actor': @@ -1569,7 +1600,8 @@ def doCast(fp, roleid, rolename): role = None order = None for item in ll[1:]: - if not item: continue + if not item: + continue if item[0] == '[': # Quite inefficient, but there are some very strange # cases of garbage in the plain text data files to handle... @@ -1581,7 +1613,8 @@ def doCast(fp, roleid, rolename): if nidx != -1: note = role[nidx:] role = role[:nidx].rstrip() - if not role: role = None + if not role: + role = None elif item[0] == '(': if note is None: note = item @@ -1590,33 +1623,34 @@ def doCast(fp, roleid, rolename): elif item[0] == '<': textor = item[1:-1] try: - order = long(textor) + order = int(textor) except ValueError: os = textor.split(',') if len(os) == 3: try: - order = ((long(os[2])-1) * 1000) + \ - ((long(os[1])-1) * 100) + (long(os[0])-1) + order = ((int(os[2]) - 1) * 1000) + \ + ((int(os[1]) - 1) * 100) + (int(os[0]) - 1) except ValueError: pass movieid = CACHE_MID.addUnique(title) if movieid is None: continue if role is not None: - roles = filter(None, [x.strip() for x in role.split('/')]) + roles = [_f for _f in [x.strip() for x in role.split('/')] if _f] for role in roles: cid = CACHE_CID.addUnique(role) sqldata.add((pid, movieid, cid, note, order)) else: sqldata.add((pid, movieid, None, note, order)) if count % 10000 == 0: - print 'SCANNING %s:' % rolename, - print _(name) + print('SCANNING %s:' % rolename, end=' ') + print(_(name)) count += 1 sqldata.flush() CACHE_PID.flush() CACHE_PID.personGender.clear() - print 'CLOSING %s...' % rolename + CACHE_CID.flush() + print('CLOSING %s...' % rolename) def castLists(): @@ -1627,10 +1661,13 @@ def castLists(): continue fname = rolename fname = fname.replace(' ', '-') - if fname == 'actress': fname = 'actresses.list.gz' - elif fname == 'miscellaneous-crew': fname = 'miscellaneous.list.gz' - else: fname = fname + 's.list.gz' - print 'DOING', fname + if fname == 'actress': + fname = 'actresses.list.gz' + elif fname == 'miscellaneous-crew': + fname = 'miscellaneous.list.gz' + else: + fname = fname + 's.list.gz' + print('DOING', fname) try: f = SourceFile(fname, start=CAST_START, stop=CAST_STOP) except IOError: @@ -1652,37 +1689,44 @@ def doAkaNames(): """People's akas.""" pid = None count = 0 - try: fp = SourceFile('aka-names.list.gz', start=AKAN_START) - except IOError: return + try: + fp = SourceFile('aka-names.list.gz', start=AKAN_START) + except IOError: + return sqldata = SQLData(table=AkaName, cols=['personID', 'name', 'imdbIndex', - 'namePcodeCf', 'namePcodeNf', 'surnamePcode', - 'md5sum']) + 'namePcodeCf', 'namePcodeNf', 'surnamePcode', + 'md5sum']) for line in fp: if line and line[0] != ' ': - if line[0] == '\n': continue + if line[0] == '\n': + continue pid = CACHE_PID.addUnique(line.strip()) else: line = line.strip() - if line[:5] == '(aka ': line = line[5:] - if line[-1:] == ')': line = line[:-1] + if line[:5] == '(aka ': + line = line[5:] + if line[-1:] == ')': + line = line[:-1] try: name_dict = analyze_name(line) except IMDbParserError: - if line: print 'WARNING doAkaNames wrong name:', _(line) + if line: + print('WARNING doAkaNames wrong name:', _(line)) continue name = name_dict.get('name') namePcodeCf, namePcodeNf, surnamePcode = name_soundexes(name) sqldata.add((pid, name, name_dict.get('imdbIndex'), namePcodeCf, namePcodeNf, surnamePcode, - md5(line).hexdigest())) + md5(line.encode('latin1')).hexdigest())) if count % 10000 == 0: - print 'SCANNING akanames:', _(line) + print('SCANNING akanames:', _(line)) count += 1 sqldata.flush() fp.close() class AkasMoviesCache(MoviesCache): + """A MoviesCache-like class used to populate the AkaTitle table.""" className = 'AkasMoviesCache' counter = counter() @@ -1696,12 +1740,11 @@ class AkasMoviesCache(MoviesCache): self._table_name = tableName(AkaTitle) self._id_for_custom_q = 'AKAMOVIES' self.sqlstr, self.converter = createSQLstr(AkaTitle, ('id', 'movieID', - 'title', 'imdbIndex', 'kindID', 'productionYear', - 'phoneticCode', 'episodeOfID', 'seasonNr', - 'episodeNr', 'note', 'md5sum')) + 'title', 'imdbIndex', 'kindID', 'productionYear', + 'phoneticCode', 'episodeOfID', 'seasonNr', + 'episodeNr', 'note', 'md5sum')) def flush(self, *args, **kwds): - # Preserve consistency of ForeignKey. CACHE_MID.flush(quiet=1) super(AkasMoviesCache, self).flush(*args, **kwds) @@ -1734,14 +1777,14 @@ def doAkaTitles(): """Movies' akas.""" mid = None count = 0 - for fname, start in (('aka-titles.list.gz',AKAT_START), - ('italian-aka-titles.list.gz',AKAT_IT_START), - ('german-aka-titles.list.gz',AKAT_DE_START), - ('iso-aka-titles.list.gz',AKAT_ISO_START), - (os.path.join('contrib','hungarian-aka-titles.list.gz'), - AKAT_HU_START), - (os.path.join('contrib','norwegian-aka-titles.list.gz'), - AKAT_NO_START)): + for fname, start in (('aka-titles.list.gz', AKAT_START), + ('italian-aka-titles.list.gz', AKAT_IT_START), + ('german-aka-titles.list.gz', AKAT_DE_START), + ('iso-aka-titles.list.gz', AKAT_ISO_START), + (os.path.join('contrib', 'hungarian-aka-titles.list.gz'), + AKAT_HU_START), + (os.path.join('contrib', 'norwegian-aka-titles.list.gz'), + AKAT_NO_START)): incontrib = 0 pwarning = 1 # Looks like that the only up-to-date AKA file is aka-titles. @@ -1764,18 +1807,19 @@ def doAkaTitles(): if line and line[0] != ' ': # Reading the official title. doNotAdd = False - if line[0] == '\n': continue + if line[0] == '\n': + continue line = line.strip() if obsolete: try: - tonD = analyze_title(line, _emptyString='') + tonD = analyze_title(line) except IMDbParserError: if line: - print 'WARNING doAkaTitles(obsol O) invalid title:', - print _(line) + print('WARNING doAkaTitles(obsol O) invalid title:', end=' ') + print(_(line)) continue tonD['title'] = normalizeTitle(tonD['title']) - line = build_title(tonD, ptdf=1, _emptyString='') + line = build_title(tonD, ptdf=True) # Aka information for titles in obsolete files are # added only if the movie already exists in the cache. if line not in CACHE_MID: @@ -1786,18 +1830,18 @@ def doAkaTitles(): continue if line[0] == '"': try: - titleDict = analyze_title(line, _emptyString='') + titleDict = analyze_title(line) except IMDbParserError: if line: - print 'WARNING doAkaTitles (O) invalid title:', - print _(line) + print('WARNING doAkaTitles (O) invalid title:', end=' ') + print(_(line)) continue if 'episode of' in titleDict: if obsolete: titleDict['episode of']['title'] = \ normalizeTitle(titleDict['episode of']['title']) series = build_title(titleDict['episode of'], - ptdf=1, _emptyString='') + ptdf=True) seriesID = CACHE_MID.addUnique(series) if seriesID is None: continue @@ -1815,44 +1859,50 @@ def doAkaTitles(): res = unpack(line.strip(), ('title', 'note')) note = res.get('note') if incontrib: - if res.get('note'): note += ' ' - else: note = '' - if start == AKAT_HU_START: note += '(Hungary)' - elif start == AKAT_NO_START: note += '(Norway)' + if res.get('note'): + note += ' ' + else: + note = '' + if start == AKAT_HU_START: + note += '(Hungary)' + elif start == AKAT_NO_START: + note += '(Norway)' akat = res.get('title', '') - if akat[:5] == '(aka ': akat = akat[5:] - if akat[-2:] in ('))', '})'): akat = akat[:-1] + if akat[:5] == '(aka ': + akat = akat[5:] + if akat[-2:] in ('))', '})'): + akat = akat[:-1] akat = akat.strip() if not akat: continue if obsolete: try: - akatD = analyze_title(akat, _emptyString='') + akatD = analyze_title(akat) except IMDbParserError: if line: - print 'WARNING doAkaTitles(obsol) invalid title:', - print _(akat) + print('WARNING doAkaTitles(obsol) invalid title:', end=' ') + print(_(akat)) continue akatD['title'] = normalizeTitle(akatD['title']) - akat = build_title(akatD, ptdf=1, _emptyString='') + akat = build_title(akatD, ptdf=True) if count % 10000 == 0: - print 'SCANNING %s:' % fname[:-8].replace('-', ' '), - print _(akat) + print('SCANNING %s:' % fname[:-8].replace('-', ' '), end=' ') + print(_(akat)) if isEpisode and seriesID is not None: # Handle series for which only single episodes have # aliases. try: - akaDict = analyze_title(akat, _emptyString='') + akaDict = analyze_title(akat) except IMDbParserError: if line: - print 'WARNING doAkaTitles (epis) invalid title:', - print _(akat) + print('WARNING doAkaTitles (epis) invalid title:', end=' ') + print(_(akat)) continue if 'episode of' in akaDict: if obsolete: akaDict['episode of']['title'] = normalizeTitle( - akaDict['episode of']['title']) - akaSeries = build_title(akaDict['episode of'], ptdf=1) + akaDict['episode of']['title']) + akaSeries = build_title(akaDict['episode of'], ptdf=True) CACHE_MID_AKAS.add(akaSeries, [('ids', seriesID)]) append_data = [('ids', mid)] if note is not None: @@ -1871,32 +1921,35 @@ def doMovieLinks(): mid = None count = 0 sqldata = SQLData(table=MovieLink, - cols=['movieID', 'linkedMovieID', 'linkTypeID'], - flushEvery=10000) - try: fp = SourceFile('movie-links.list.gz', start=LINK_START) - except IOError: return + cols=['movieID', 'linkedMovieID', 'linkTypeID'], + flushEvery=10000) + try: + fp = SourceFile('movie-links.list.gz', start=LINK_START) + except IOError: + return for line in fp: if line and line[0] != ' ': - if line[0] == '\n': continue + if line[0] == '\n': + continue title = line.strip() mid = CACHE_MID.addUnique(title) if mid is None: continue if count % 10000 == 0: - print 'SCANNING movielinks:', _(title) + print('SCANNING movielinks:', _(title)) else: - if mid == None: - continue - line = line.strip() - link_txt = unicode(line, 'utf_8').encode('ascii', 'replace') + if mid is None: + continue + link_txt = line = line.strip() theid = None for k, lenkp1, v in MOVIELINK_IDS: if link_txt and link_txt[0] == '(' \ - and link_txt[1:lenkp1+1] == k: + and link_txt[1:lenkp1 + 1] == k: theid = v break - if theid is None: continue - totitle = line[lenkp1+2:-1].strip() + if theid is None: + continue + totitle = line[lenkp1 + 2:-1].strip() totitleid = CACHE_MID.addUnique(totitle) if totitleid is None: continue @@ -1909,29 +1962,32 @@ def doMovieLinks(): def minusHashFiles(fp, funct, defaultid, descr): """A file with lines starting with '# ' and '- '.""" sqldata = SQLData(table=MovieInfo, - cols=['movieID', 'infoTypeID', 'info', 'note']) + cols=['movieID', 'infoTypeID', 'info', 'note']) sqldata.flushEvery = 2500 - if descr == 'quotes': sqldata.flushEvery = 4000 - elif descr == 'soundtracks': sqldata.flushEvery = 3000 - elif descr == 'trivia': sqldata.flushEvery = 3000 + if descr == 'quotes': + sqldata.flushEvery = 4000 + elif descr == 'soundtracks': + sqldata.flushEvery = 3000 + elif descr == 'trivia': + sqldata.flushEvery = 3000 count = 0 for title, text in fp.getByHashSections(): title = title.strip() d = funct(text.split('\n')) if not d: - print 'WARNING skipping empty information about title:', - print _(title) + print('WARNING skipping empty information about title:', end=' ') + print(_(title)) continue if not title: - print 'WARNING skipping information associated to empty title:', - print _(d[0], truncateAt=40) + print('WARNING skipping information associated to empty title:', end=' ') + print(_(d[0], truncateAt=40)) continue mid = CACHE_MID.addUnique(title) if mid is None: continue if count % 5000 == 0: - print 'SCANNING %s:' % descr, - print _(title) + print('SCANNING %s:' % descr, end=' ') + print(_(title)) for data in d: sqldata.add((mid, defaultid, data, None)) count += 1 @@ -1940,31 +1996,35 @@ def minusHashFiles(fp, funct, defaultid, def doMinusHashFiles(): """Files with lines starting with '# ' and '- '.""" - for fname, start in [('alternate versions',AV_START), - ('goofs',GOOFS_START), ('crazy credits',CC_START), - ('quotes',QUOTES_START), - ('soundtracks',SNDT_START), - ('trivia',TRIV_START)]: + for fname, start in [('alternate versions', AV_START), + ('goofs', GOOFS_START), ('crazy credits', CC_START), + ('quotes', QUOTES_START), + ('soundtracks', SNDT_START), + ('trivia', TRIV_START)]: try: - fp = SourceFile(fname.replace(' ', '-')+'.list.gz', start=start, - stop=MINHASH_STOP) + fp = SourceFile(fname.replace(' ', '-') + '.list.gz', start=start, + stop=MINHASH_STOP) except IOError: continue funct = _parseMinusList - if fname == 'quotes': funct = getQuotes + if fname == 'quotes': + funct = getQuotes index = fname - if index == 'soundtracks': index = 'soundtrack' + if index == 'soundtracks': + index = 'soundtrack' minusHashFiles(fp, funct, INFO_TYPES[index], fname) fp.close() def getTaglines(): """Movie's taglines.""" - try: fp = SourceFile('taglines.list.gz', start=TAGL_START, stop=TAGL_STOP) - except IOError: return + try: + fp = SourceFile('taglines.list.gz', start=TAGL_START, stop=TAGL_STOP) + except IOError: + return sqldata = SQLData(table=MovieInfo, - cols=['movieID', 'infoTypeID', 'info', 'note'], - flushEvery=10000) + cols=['movieID', 'infoTypeID', 'info', 'note'], + flushEvery=10000) count = 0 for title, text in fp.getByHashSections(): title = title.strip() @@ -1973,9 +2033,10 @@ def getTaglines(): continue for tag in text.split('\n'): tag = tag.strip() - if not tag: continue + if not tag: + continue if count % 10000 == 0: - print 'SCANNING taglines:', _(title) + print('SCANNING taglines:', _(title)) sqldata.add((mid, INFO_TYPES['taglines'], tag, None)) count += 1 sqldata.flush() @@ -1990,14 +2051,18 @@ def getQuotes(lines): if line and line[:2] == ' ' and qttl and qttl[-1] and \ not qttl[-1].endswith('::'): line = line.lstrip() - if line: qttl[-1] += ' %s' % line + if line: + qttl[-1] += ' %s' % line elif not line.strip(): - if qttl: quotes.append('::'.join(qttl)) + if qttl: + quotes.append('::'.join(qttl)) qttl[:] = [] else: line = line.lstrip() - if line: qttl.append(line) - if qttl: quotes.append('::'.join(qttl)) + if line: + qttl.append(line) + if qttl: + quotes.append('::'.join(qttl)) return quotes @@ -2011,17 +2076,19 @@ _bus = {'BT': 'budget', 'PD': 'production dates', 'ST': 'studios', 'CP': 'copyright holder' -} + } _usd = '$' -_gbp = unichr(0x00a3).encode('utf_8') -_eur = unichr(0x20ac).encode('utf_8') +_gbp = chr(0x00a3) +_eur = chr(0x20ac) + + def getBusiness(lines): """Movie's business information.""" bd = _parseColonList(lines, _bus) - for k in bd.keys(): + for k in list(bd.keys()): nv = [] for v in bd[k]: - v = v.replace('USD ',_usd).replace('GBP ',_gbp).replace('EUR',_eur) + v = v.replace('USD ', _usd).replace('GBP ', _gbp).replace('EUR', _eur) nv.append(v) bd[k] = nv return bd @@ -2076,15 +2143,16 @@ _ldk = {'OT': 'original title', 'LB': 'label', 'CN': 'catalog number', 'LT': 'laserdisc title' -} + } # Handle laserdisc keys. -for key, value in _ldk.items(): +for key, value in list(_ldk.items()): _ldk[key] = 'LD %s' % value + def getLaserDisc(lines): """Laserdisc information.""" d = _parseColonList(lines, _ldk) - for k, v in d.iteritems(): + for k, v in d.items(): d[k] = ' '.join(v) return d @@ -2098,23 +2166,28 @@ _lit = {'SCRP': 'screenplay-teleplay', 'CRIT': 'printed media reviews', 'ESSY': 'essays', 'OTHR': 'other literature' -} + } + + def getLiterature(lines): """Movie's literature information.""" return _parseColonList(lines, _lit) _mpaa = {'RE': 'mpaa'} + + def getMPAA(lines): """Movie's mpaa information.""" d = _parseColonList(lines, _mpaa) - for k, v in d.iteritems(): + for k, v in d.items(): d[k] = ' '.join(v) return d re_nameImdbIndex = re.compile(r'\(([IVXLCDM]+)\)') + def nmmvFiles(fp, funct, fname): """Files with sections separated by 'MV: ' or 'NM: '.""" count = 0 @@ -2127,34 +2200,39 @@ def nmmvFiles(fp, funct, fname): guestid = RoleType.select(RoleType.q.role == 'guest')[0].id roleid = str(guestid) guestdata = SQLData(table=CastInfo, - cols=['personID', 'movieID', 'personRoleID', 'note', - RawValue('roleID', roleid)], flushEvery=10000) + cols=['personID', 'movieID', 'personRoleID', 'note', + RawValue('roleID', roleid)], flushEvery=10000) akanamesdata = SQLData(table=AkaName, cols=['personID', 'name', - 'imdbIndex', 'namePcodeCf', 'namePcodeNf', 'surnamePcode', - 'md5sum']) + 'imdbIndex', 'namePcodeCf', 'namePcodeNf', 'surnamePcode', + 'md5sum']) else: datakind = 'movie' sqls = sqlsM guestdata = None akanamesdata = None sqldata = SQLData(table=sqls[0], cols=sqls[1]) - if fname == 'plot.list.gz': sqldata.flushEvery = 1100 - elif fname == 'literature.list.gz': sqldata.flushEvery = 5000 - elif fname == 'business.list.gz': sqldata.flushEvery = 10000 - elif fname == 'biographies.list.gz': sqldata.flushEvery = 5000 + if fname == 'plot.list.gz': + sqldata.flushEvery = 1100 + elif fname == 'literature.list.gz': + sqldata.flushEvery = 5000 + elif fname == 'business.list.gz': + sqldata.flushEvery = 10000 + elif fname == 'biographies.list.gz': + sqldata.flushEvery = 5000 islaserdisc = False if fname == 'laserdisc.list.gz': islaserdisc = True _ltype = type([]) for ton, text in fp.getByNMMVSections(): ton = ton.strip() - if not ton: continue + if not ton: + continue note = None if datakind == 'movie': if islaserdisc: - tonD = analyze_title(ton, _emptyString='') + tonD = analyze_title(ton) tonD['title'] = normalizeTitle(tonD['title']) - ton = build_title(tonD, ptdf=1, _emptyString='') + ton = build_title(tonD, ptdf=True) # Skips movies that are not already in the cache, since # laserdisc.list.gz is an obsolete file. if ton not in CACHE_MID: @@ -2162,18 +2240,19 @@ def nmmvFiles(fp, funct, fname): mopid = CACHE_MID.addUnique(ton) if mopid is None: continue - else: mopid = CACHE_PID.addUnique(ton) + else: + mopid = CACHE_PID.addUnique(ton) if count % 6000 == 0: - print 'SCANNING %s:' % fname[:-8].replace('-', ' '), - print _(ton) + print('SCANNING %s:' % fname[:-8].replace('-', ' '), end=' ') + print(_(ton)) d = funct(text.split('\n')) - for k, v in d.iteritems(): + for k, v in d.items(): if k != 'notable tv guest appearances': theid = INFO_TYPES.get(k) if theid is None: - print 'WARNING key "%s" of ToN' % k, - print _(ton), - print 'not in INFO_TYPES' + print('WARNING key "%s" of ToN' % k, end=' ') + print(_(ton), end=' ') + print('not in INFO_TYPES') continue if type(v) is _ltype: for i in v: @@ -2182,37 +2261,37 @@ def nmmvFiles(fp, funct, fname): # are a list of Movie object (yes, imdb.Movie.Movie) # FIXME: no more used? title = i.get('long imdb canonical title') - if not title: continue + if not title: + continue movieid = CACHE_MID.addUnique(title) if movieid is None: continue crole = i.currentRole if isinstance(crole, list): - crole = ' / '.join([x.get('long imdb name', u'') + crole = ' / '.join([x.get('long imdb name', '') for x in crole]) if not crole: crole = None - else: - crole = unicode(crole).encode('utf_8') guestdata.add((mopid, movieid, crole, - i.notes or None)) + i.notes or None)) continue if k in ('plot', 'mini biography'): s = i.split('::') if len(s) == 2: - #if note: note += ' ' - #else: note = '' - #note += '(author: %s)' % s[1] note = s[1] i = s[0] - if i: sqldata.add((mopid, theid, i, note)) + if i: + sqldata.add((mopid, theid, i, note)) note = None else: - if v: sqldata.add((mopid, theid, v, note)) + if v: + sqldata.add((mopid, theid, v, note)) if k in ('nick names', 'birth name') and v: # Put also the birth name/nick names in the list of aliases. - if k == 'birth name': realnames = [v] - else: realnames = v + if k == 'birth name': + realnames = [v] + else: + realnames = v for realname in realnames: imdbIndex = re_nameImdbIndex.findall(realname) or None if imdbIndex: @@ -2220,18 +2299,20 @@ def nmmvFiles(fp, funct, fname): realname = re_nameImdbIndex.sub('', realname) if realname: # XXX: check for duplicates? - ##if k == 'birth name': - ## realname = canonicalName(realname) - ##else: - ## realname = normalizeName(realname) + # if k == 'birth name': + # realname = canonicalName(realname) + # else: + # realname = normalizeName(realname) namePcodeCf, namePcodeNf, surnamePcode = \ - name_soundexes(realname) + name_soundexes(realname) akanamesdata.add((mopid, realname, imdbIndex, - namePcodeCf, namePcodeNf, surnamePcode, - md5(realname).hexdigest())) + namePcodeCf, namePcodeNf, surnamePcode, + md5(realname.encode('latin1')).hexdigest())) count += 1 - if guestdata is not None: guestdata.flush() - if akanamesdata is not None: akanamesdata.flush() + if guestdata is not None: + guestdata.flush() + if akanamesdata is not None: + akanamesdata.flush() sqldata.flush() @@ -2262,16 +2343,19 @@ def _parseList(l, prefix, mline=1): reslapp(joiner(ltmp)) ltmp[:] = [] data = line[firstlen:].strip() - if data: ltmpapp(data) + if data: + ltmpapp(data) elif mline and line[:otherlen] == otherl: data = line[otherlen:].strip() - if data: ltmpapp(data) + if data: + ltmpapp(data) else: if ltmp: reslapp(joiner(ltmp)) ltmp[:] = [] if parsing: - if ltmp: reslapp(joiner(ltmp)) + if ltmp: + reslapp(joiner(ltmp)) break return resl @@ -2301,7 +2385,8 @@ def _parseBiography(biol): res = {} bio = ' '.join(_parseList(biol, 'BG', mline=0)) bio = _parseBioBy(biol) - if bio: res['mini biography'] = bio + if bio: + res['mini biography'] = bio for x in biol: x4 = x[:4] @@ -2322,12 +2407,14 @@ def _parseBiography(biol): res.setdefault('spouse', []).append(x[6:].strip()) elif x4 == 'RN: ': n = x[4:].strip() - if not n: continue + if not n: + continue try: rn = build_name(analyze_name(n, canonical=1), canonical=1) res['birth name'] = rn except IMDbParserError: - if line: print 'WARNING _parseBiography wrong name:', _(n) + if line: + print('WARNING _parseBiography wrong name:', _(n)) continue elif x6 == 'AT: * ': res.setdefault('article', []).append(x[6:].strip()) @@ -2345,23 +2432,32 @@ def _parseBiography(biol): sal = x[6:].strip().replace(' -> ', '::') res.setdefault('salary history', []).append(sal) trl = _parseList(biol, 'TR') - if trl: res['trivia'] = trl + if trl: + res['trivia'] = trl quotes = _parseList(biol, 'QU') - if quotes: res['quotes'] = quotes + if quotes: + res['quotes'] = quotes otherworks = _parseList(biol, 'OW') - if otherworks: res['other works'] = otherworks + if otherworks: + res['other works'] = otherworks books = _parseList(biol, 'BO') - if books: res['books'] = books + if books: + res['books'] = books agent = _parseList(biol, 'AG') - if agent: res['agent address'] = agent + if agent: + res['agent address'] = agent wherenow = _parseList(biol, 'WN') - if wherenow: res['where now'] = wherenow[0] + if wherenow: + res['where now'] = wherenow[0] biomovies = _parseList(biol, 'BT') - if biomovies: res['biographical movies'] = biomovies + if biomovies: + res['biographical movies'] = biomovies tm = _parseList(biol, 'TM') - if tm: res['trade mark'] = tm + if tm: + res['trade mark'] = tm interv = _parseList(biol, 'IT') - if interv: res['interviews'] = interv + if interv: + res['interviews'] = interv return res # ============ @@ -2376,13 +2472,14 @@ def doNMMVFiles(): ('literature.list.gz', LIT_START, getLiterature), ('mpaa-ratings-reasons.list.gz', MPAA_START, getMPAA), ('plot.list.gz', PLOT_START, getPlot)]: - ##for fname, start, funct in [('business.list.gz',BUS_START,getBusiness)]: try: fp = SourceFile(fname, start=start) except IOError: continue - if fname == 'literature.list.gz': fp.set_stop(LIT_STOP) - elif fname == 'business.list.gz': fp.set_stop(BUS_STOP) + if fname == 'literature.list.gz': + fp.set_stop(LIT_STOP) + elif fname == 'business.list.gz': + fp.set_stop(BUS_STOP) nmmvFiles(fp, funct, fname) fp.close() t('doNMMVFiles(%s)' % fname[:-8].replace('-', ' ')) @@ -2392,22 +2489,24 @@ def doMovieCompaniesInfo(): """Files with information on a single line about movies, concerning companies.""" sqldata = SQLData(table=MovieCompanies, - cols=['movieID', 'companyID', 'companyTypeID', 'note']) + cols=['movieID', 'companyID', 'companyTypeID', 'note']) for dataf in (('distributors.list.gz', DIS_START), - ('miscellaneous-companies.list.gz', MIS_START), - ('production-companies.list.gz', PRO_START), - ('special-effects-companies.list.gz', SFX_START)): + ('miscellaneous-companies.list.gz', MIS_START), + ('production-companies.list.gz', PRO_START), + ('special-effects-companies.list.gz', SFX_START)): try: fp = SourceFile(dataf[0], start=dataf[1]) except IOError: continue typeindex = dataf[0][:-8].replace('-', ' ') - infoid = COMP_TYPES[typeindex] + infoid = COMP_TYPES[typeindex] count = 0 for line in fp: data = unpack(line.strip(), ('title', 'company', 'note')) - if 'title' not in data: continue - if 'company' not in data: continue + if 'title' not in data: + continue + if 'company' not in data: + continue title = data['title'] company = data['company'] mid = CACHE_MID.addUnique(title) @@ -2418,8 +2517,8 @@ def doMovieCompaniesInfo(): if 'note' in data: note = data['note'] if count % 10000 == 0: - print 'SCANNING %s:' % dataf[0][:-8].replace('-', ' '), - print _(data['title']) + print('SCANNING %s:' % dataf[0][:-8].replace('-', ' '), end=' ') + print(_(data['title'])) sqldata.add((mid, cid, infoid, note)) count += 1 sqldata.flush() @@ -2430,32 +2529,35 @@ def doMovieCompaniesInfo(): def doMiscMovieInfo(): """Files with information on a single line about movies.""" - for dataf in (('certificates.list.gz',CER_START), - ('color-info.list.gz',COL_START), - ('countries.list.gz',COU_START), - ('genres.list.gz',GEN_START), - ('keywords.list.gz',KEY_START), - ('language.list.gz',LAN_START), - ('locations.list.gz',LOC_START), - ('running-times.list.gz',RUN_START), - ('sound-mix.list.gz',SOU_START), - ('technical.list.gz',TCN_START), - ('release-dates.list.gz',RELDATE_START)): + for dataf in (('certificates.list.gz', CER_START), + ('color-info.list.gz', COL_START), + ('countries.list.gz', COU_START), + ('genres.list.gz', GEN_START), + ('keywords.list.gz', KEY_START), + ('language.list.gz', LAN_START), + ('locations.list.gz', LOC_START), + ('running-times.list.gz', RUN_START), + ('sound-mix.list.gz', SOU_START), + ('technical.list.gz', TCN_START), + ('release-dates.list.gz', RELDATE_START)): try: fp = SourceFile(dataf[0], start=dataf[1]) except IOError: continue typeindex = dataf[0][:-8].replace('-', ' ') - if typeindex == 'running times': typeindex = 'runtimes' - elif typeindex == 'technical': typeindex = 'tech info' - elif typeindex == 'language': typeindex = 'languages' + if typeindex == 'running times': + typeindex = 'runtimes' + elif typeindex == 'technical': + typeindex = 'tech info' + elif typeindex == 'language': + typeindex = 'languages' if typeindex != 'keywords': sqldata = SQLData(table=MovieInfo, - cols=['movieID', 'infoTypeID', 'info', 'note']) + cols=['movieID', 'infoTypeID', 'info', 'note']) else: sqldata = SQLData(table=MovieKeyword, - cols=['movieID', 'keywordID']) - infoid = INFO_TYPES[typeindex] + cols=['movieID', 'keywordID']) + infoid = INFO_TYPES[typeindex] count = 0 if dataf[0] == 'locations.list.gz': sqldata.flushEvery = 10000 @@ -2463,8 +2565,10 @@ def doMiscMovieInfo(): sqldata.flushEvery = 20000 for line in fp: data = unpack(line.strip(), ('title', 'info', 'note')) - if 'title' not in data: continue - if 'info' not in data: continue + if 'title' not in data: + continue + if 'info' not in data: + continue title = data['title'] mid = CACHE_MID.addUnique(title) if mid is None: @@ -2473,8 +2577,8 @@ def doMiscMovieInfo(): if 'note' in data: note = data['note'] if count % 10000 == 0: - print 'SCANNING %s:' % dataf[0][:-8].replace('-', ' '), - print _(data['title']) + print('SCANNING %s:' % dataf[0][:-8].replace('-', ' '), end=' ') + print(_(data['title'])) info = data['info'] if typeindex == 'keywords': keywordID = CACHE_KWRDID.addUnique(info) @@ -2492,21 +2596,24 @@ def doMiscMovieInfo(): def getRating(): """Movie's rating.""" - try: fp = SourceFile('ratings.list.gz', start=RAT_START, stop=RAT_STOP) - except IOError: return - sqldata = SQLData(table=MovieInfoIdx, cols=['movieID', 'infoTypeID', - 'info', 'note']) + try: + fp = SourceFile('ratings.list.gz', start=RAT_START, stop=RAT_STOP) + except IOError: + return + sqldata = SQLData(table=MovieInfo, cols=['movieID', 'infoTypeID', + 'info', 'note']) count = 0 for line in fp: data = unpack(line, ('votes distribution', 'votes', 'rating', 'title'), - sep=' ') - if 'title' not in data: continue + sep=' ') + if 'title' not in data: + continue title = data['title'].strip() mid = CACHE_MID.addUnique(title) if mid is None: continue if count % 10000 == 0: - print 'SCANNING rating:', _(title) + print('SCANNING rating:', _(title)) sqldata.add((mid, INFO_TYPES['votes distribution'], data.get('votes distribution'), None)) sqldata.add((mid, INFO_TYPES['votes'], data.get('votes'), None)) @@ -2519,26 +2626,33 @@ def getRating(): def getTopBottomRating(): """Movie's rating, scanning for top 250 and bottom 10.""" for what in ('top 250 rank', 'bottom 10 rank'): - if what == 'top 250 rank': st = RAT_TOP250_START - else: st = RAT_BOT10_START - try: fp = SourceFile('ratings.list.gz', start=st, stop=TOPBOT_STOP) - except IOError: break - sqldata = SQLData(table=MovieInfoIdx, - cols=['movieID', - RawValue('infoTypeID', INFO_TYPES[what]), - 'info', 'note']) + if what == 'top 250 rank': + st = RAT_TOP250_START + else: + st = RAT_BOT10_START + try: + fp = SourceFile('ratings.list.gz', start=st, stop=TOPBOT_STOP) + except IOError: + break + sqldata = SQLData(table=MovieInfo, + cols=['movieID', + RawValue('infoTypeID', INFO_TYPES[what]), + 'info', 'note']) count = 1 - print 'SCANNING %s...' % what + print('SCANNING %s...' % what) for line in fp: data = unpack(line, ('votes distribution', 'votes', 'rank', - 'title'), sep=' ') - if 'title' not in data: continue + 'title'), sep=' ') + if 'title' not in data: + continue title = data['title'].strip() mid = CACHE_MID.addUnique(title) if mid is None: continue - if what == 'top 250 rank': rank = count - else: rank = 11 - count + if what == 'top 250 rank': + rank = count + else: + rank = 11 - count sqldata.add((mid, str(rank), None)) count += 1 sqldata.flush() @@ -2567,29 +2681,32 @@ def completeCast(): cckinds = [(x.id, x.kind) for x in CompCastType.select()] for k, v in cckinds: CCKind[v] = k - for fname, start in [('complete-cast.list.gz',COMPCAST_START), - ('complete-crew.list.gz',COMPCREW_START)]: + for fname, start in [('complete-cast.list.gz', COMPCAST_START), + ('complete-crew.list.gz', COMPCREW_START)]: try: fp = SourceFile(fname, start=start, stop=COMP_STOP) except IOError: continue - if fname == 'complete-cast.list.gz': obj = 'cast' - else: obj = 'crew' + if fname == 'complete-cast.list.gz': + obj = 'cast' + else: + obj = 'crew' subID = str(CCKind[obj]) sqldata = SQLData(table=CompleteCast, - cols=['movieID', RawValue('subjectID', subID), - 'statusID']) + cols=['movieID', RawValue('subjectID', subID), + 'statusID']) count = 0 for line in fp: ll = [x for x in line.split('\t') if x] - if len(ll) != 2: continue + if len(ll) != 2: + continue title = ll[0] mid = CACHE_MID.addUnique(title) if mid is None: continue if count % 10000 == 0: - print 'SCANNING %s:' % fname[:-8].replace('-', ' '), - print _(title) + print('SCANNING %s:' % fname[:-8].replace('-', ' '), end=' ') + print(_(title)) sqldata.add((mid, CCKind[ll[1].lower().strip()])) count += 1 fp.close() @@ -2603,15 +2720,6 @@ CACHE_CID = CharactersCache() CACHE_CID.className = 'CharactersCache' CACHE_COMPID = CompaniesCache() CACHE_KWRDID = KeywordsCache() - -def _cmpfunc(x, y): - """Sort a list of tuples, by the length of the first item (in reverse).""" - lx = len(x[0]) - ly = len(y[0]) - if lx > ly: return -1 - elif lx < ly: return 1 - return 0 - INFO_TYPES = {} MOVIELINK_IDS = [] KIND_IDS = {} @@ -2619,17 +2727,18 @@ KIND_STRS = {} CCAST_TYPES = {} COMP_TYPES = {} + def readConstants(): """Read constants from the database.""" global INFO_TYPES, MOVIELINK_IDS, KIND_IDS, KIND_STRS, \ - CCAST_TYPES, COMP_TYPES + CCAST_TYPES, COMP_TYPES for x in InfoType.select(): INFO_TYPES[x.info] = x.id for x in LinkType.select(): MOVIELINK_IDS.append((x.link, len(x.link), x.id)) - MOVIELINK_IDS.sort(_cmpfunc) + MOVIELINK_IDS.sort(key=lambda x: operator.length_hint(x[0]), reverse=True) for x in KindType.select(): KIND_IDS[x.kind] = x.id @@ -2645,7 +2754,7 @@ def readConstants(): def _imdbIDsFileName(fname): """Return a file name, adding the optional CSV_DIR directory.""" - return os.path.join(*(filter(None, [CSV_DIR, fname]))) + return os.path.join(*([_f for _f in [CSV_DIR, fname] if _f])) def _countRows(tableName): @@ -2653,8 +2762,8 @@ def _countRows(tableName): try: CURS.execute('SELECT COUNT(*) FROM %s' % tableName) return (CURS.fetchone() or [0])[0] - except Exception, e: - print 'WARNING: unable to count rows of table %s: %s' % (tableName, e) + except Exception as e: + print('WARNING: unable to count rows of table %s: %s' % (tableName, e)) return 0 @@ -2662,15 +2771,19 @@ def storeNotNULLimdbIDs(cls): """Store in a temporary table or in a dbm database a mapping between md5sum (of title or name) and imdbID, when the latter is present in the database.""" - if cls is Title: cname = 'movies' - elif cls is Name: cname = 'people' - elif cls is CompanyName: cname = 'companies' - else: cname = 'characters' + if cls is Title: + cname = 'movies' + elif cls is Name: + cname = 'people' + elif cls is CompanyName: + cname = 'companies' + else: + cname = 'characters' table_name = tableName(cls) md5sum_col = colName(cls, 'md5sum') imdbID_col = colName(cls, 'imdbID') - print 'SAVING imdbID values for %s...' % cname, + print('SAVING imdbID values for %s...' % cname, end=' ') sys.stdout.flush() if _get_imdbids_method() == 'table': try: @@ -2680,36 +2793,36 @@ def storeNotNULLimdbIDs(cls): pass try: CURS.execute('SELECT * FROM %s LIMIT 1' % table_name) - except Exception, e: - print 'missing "%s" table (ok if this is the first run)' % table_name + except Exception as e: + print('missing "%s" table (ok if this is the first run)' % table_name) return query = 'CREATE TEMPORARY TABLE %s_extract AS SELECT %s, %s FROM %s WHERE %s IS NOT NULL' % \ (table_name, md5sum_col, imdbID_col, - table_name, imdbID_col) + table_name, imdbID_col) CURS.execute(query) CURS.execute('CREATE INDEX %s_md5sum_idx ON %s_extract (%s)' % (table_name, table_name, md5sum_col)) CURS.execute('CREATE INDEX %s_imdbid_idx ON %s_extract (%s)' % (table_name, table_name, imdbID_col)) rows = _countRows('%s_extract' % table_name) - print 'DONE! (%d entries using a temporary table)' % rows + print('DONE! (%d entries using a temporary table)' % rows) return - except Exception, e: - print 'WARNING: unable to store imdbIDs in a temporary table (falling back to dbm): %s' % e + except Exception as e: + print('WARNING: unable to store imdbIDs in a temporary table (falling back to dbm): %s' % e) try: - db = anydbm.open(_imdbIDsFileName('%s_imdbIDs.db' % cname), 'c') - except Exception, e: - print 'WARNING: unable to store imdbIDs: %s' % str(e) + db = dbm.open(_imdbIDsFileName('%s_imdbIDs.db' % cname), 'c') + except Exception as e: + print('WARNING: unable to store imdbIDs: %s' % str(e)) return try: CURS.execute('SELECT %s, %s FROM %s WHERE %s IS NOT NULL' % - (md5sum_col, imdbID_col, table_name, imdbID_col)) + (md5sum_col, imdbID_col, table_name, imdbID_col)) res = CURS.fetchmany(10000) while res: db.update(dict((str(x[0]), str(x[1])) for x in res)) res = CURS.fetchmany(10000) - except Exception, e: - print 'SKIPPING: unable to retrieve data: %s' % e + except Exception as e: + print('SKIPPING: unable to retrieve data: %s' % e) return - print 'DONE! (%d entries)' % len(db) + print('DONE! (%d entries)' % len(db)) db.close() return @@ -2719,7 +2832,7 @@ def iterbatch(iterable, size): sourceiter = iter(iterable) while True: batchiter = islice(sourceiter, size) - yield chain([batchiter.next()], batchiter) + yield chain([next(batchiter)], batchiter) def restoreImdbIDs(cls): @@ -2732,7 +2845,7 @@ def restoreImdbIDs(cls): cname = 'companies' else: cname = 'characters' - print 'RESTORING imdbIDs values for %s...' % cname, + print('RESTORING imdbIDs values for %s...' % cname, end=' ') sys.stdout.flush() table_name = tableName(cls) md5sum_col = colName(cls, 'md5sum') @@ -2742,43 +2855,46 @@ def restoreImdbIDs(cls): try: try: CURS.execute('SELECT * FROM %s_extract LIMIT 1' % table_name) - except Exception, e: + except Exception as e: raise Exception('missing "%s_extract" table (ok if this is the first run)' % table_name) if DB_NAME == 'mysql': query = 'UPDATE %s INNER JOIN %s_extract USING (%s) SET %s.%s = %s_extract.%s' % \ (table_name, table_name, md5sum_col, - table_name, imdbID_col, table_name, imdbID_col) + table_name, imdbID_col, table_name, imdbID_col) else: query = 'UPDATE %s SET %s = %s_extract.%s FROM %s_extract WHERE %s.%s = %s_extract.%s' % \ (table_name, imdbID_col, table_name, - imdbID_col, table_name, table_name, - md5sum_col, table_name, md5sum_col) + imdbID_col, table_name, table_name, + md5sum_col, table_name, md5sum_col) CURS.execute(query) affected_rows = 'an unknown number of' try: CURS.execute('SELECT COUNT(*) FROM %s WHERE %s IS NOT NULL' % - (table_name, imdbID_col)) + (table_name, imdbID_col)) affected_rows = (CURS.fetchone() or [0])[0] - except Exception, e: + except Exception as e: pass rows = _countRows('%s_extract' % table_name) - print 'DONE! (restored %s entries out of %d)' % (affected_rows, rows) + print('DONE! (restored %s entries out of %d)' % (affected_rows, rows)) t('restore %s' % cname) - try: CURS.execute('DROP TABLE %s_extract' % table_name) - except: pass + try: + CURS.execute('DROP TABLE %s_extract' % table_name) + except: + pass return - except Exception, e: - print 'WARNING: unable to restore imdbIDs using the temporary table (falling back to dbm): %s' % e + except Exception as e: + print('INFO: unable to restore imdbIDs using the temporary table (falling back to dbm): %s' % e) try: - db = anydbm.open(_imdbIDsFileName('%s_imdbIDs.db' % cname), 'r') - except Exception, e: - print 'WARNING: unable to restore imdbIDs (ok if this is the first run)' + db = dbm.open(_imdbIDsFileName('%s_imdbIDs.db' % cname), 'r') + except Exception as e: + print('INFO: unable to restore imdbIDs (ok if this is the first run)') return count = 0 sql = "UPDATE " + table_name + " SET " + imdbID_col + \ - " = CASE " + md5sum_col + " %s END WHERE " + \ - md5sum_col + " IN (%s)" + " = CASE " + md5sum_col + " %s END WHERE " + \ + md5sum_col + " IN (%s)" + def _restore(query, batch): """Execute a query to restore a batch of imdbIDs""" items = list(batch) @@ -2788,25 +2904,26 @@ def restoreImdbIDs(cls): if success: return len(items) return 0 - for batch in iterbatch(db.iteritems(), 10000): + for batch in iterbatch(iter(db.items()), 10000): count += _restore(sql, batch) - print 'DONE! (restored %d entries out of %d)' % (count, len(db)) + print('DONE! (restored %d entries out of %d)' % (count, len(db))) t('restore %s' % cname) db.close() return + def restoreAll_imdbIDs(): """Restore imdbIDs for movies, persons, companies and characters.""" # Restoring imdbIDs for movies and persons (moved after the # built of indexes, so that it can take advantage of them). runSafely(restoreImdbIDs, 'failed to restore imdbIDs for movies', - None, Title) + None, Title) runSafely(restoreImdbIDs, 'failed to restore imdbIDs for people', - None, Name) + None, Name) runSafely(restoreImdbIDs, 'failed to restore imdbIDs for characters', - None, CharName) + None, CharName) runSafely(restoreImdbIDs, 'failed to restore imdbIDs for companies', - None, CompanyName) + None, CompanyName) def runSafely(funct, fmsg, default, *args, **kwds): @@ -2816,8 +2933,8 @@ def runSafely(funct, fmsg, default, *arg value of the function is returned (or 'default').""" try: return funct(*args, **kwds) - except Exception, e: - print 'WARNING: %s: %s' % (fmsg, e) + except Exception as e: + print('WARNING: %s: %s' % (fmsg, e)) return default @@ -2827,22 +2944,23 @@ def _executeQuery(query): s_query = query[:60] + '...' else: s_query = query - print 'EXECUTING "%s"...' % (s_query), + print('EXECUTING "%s"...' % s_query, end=' ') sys.stdout.flush() try: CURS.execute(query) - print 'DONE!' + print('DONE!') return True - except Exception, e: - print 'FAILED (%s)!' % e + except Exception as e: + print('FAILED (%s)!' % e) return False def executeCustomQueries(when, _keys=None, _timeit=True): """Run custom queries as specified on the command line.""" - if _keys is None: _keys = {} + if _keys is None: + _keys = {} for query in CUSTOM_QUERIES.get(when, []): - print 'EXECUTING "%s:%s"...' % (when, query) + print('EXECUTING "%s:%s"...' % (when, query)) sys.stdout.flush() if query.startswith('FOR_EVERY_TABLE:'): query = query[16:] @@ -2855,42 +2973,36 @@ def executeCustomQueries(when, _keys=Non _executeQuery(query % keys) if _timeit: t('%s command' % when) - except Exception, e: - print 'FAILED (%s)!' % e + except Exception as e: + print('FAILED (%s)!' % e) continue else: try: _executeQuery(query % _keys) - except Exception, e: - print 'FAILED (%s)!' % e + except Exception as e: + print('FAILED (%s)!' % e) continue if _timeit: t('%s command' % when) def buildIndexesAndFK(): - """Build indexes and Foreign Keys.""" + """Build indexes.""" executeCustomQueries('BEFORE_INDEXES') - print 'building database indexes (this may take a while)' + print('building database indexes (this may take a while)') sys.stdout.flush() # Build database indexes. idx_errors = createIndexes(DB_TABLES) for idx_error in idx_errors: - print 'ERROR caught exception creating an index: %s' % idx_error + print('ERROR caught exception creating an index: %s' % idx_error) t('createIndexes()') - print 'adding foreign keys (this may take a while)' sys.stdout.flush() - # Add FK. - fk_errors = createForeignKeys(DB_TABLES) - for fk_error in fk_errors: - print 'ERROR caught exception creating a foreign key: %s' % fk_error - t('createForeignKeys()') def restoreCSV(): """Only restore data from a set of CSV files.""" CSV_CURS.buildFakeFileNames() - print 'loading CSV files into the database' + print('loading CSV files into the database') executeCustomQueries('BEFORE_CSV_LOAD') loadCSVFiles() t('loadCSVFiles()') @@ -2904,32 +3016,32 @@ def restoreCSV(): # begin the iterations... def run(): - print 'RUNNING imdbpy2sql.py using the %s ORM' % USED_ORM + print('RUNNING imdbpy2sql.py') executeCustomQueries('BEGIN') # Storing imdbIDs for movies and persons. runSafely(storeNotNULLimdbIDs, 'failed to read imdbIDs for movies', - None, Title) + None, Title) runSafely(storeNotNULLimdbIDs, 'failed to read imdbIDs for people', - None, Name) + None, Name) runSafely(storeNotNULLimdbIDs, 'failed to read imdbIDs for characters', - None, CharName) + None, CharName) runSafely(storeNotNULLimdbIDs, 'failed to read imdbIDs for companies', - None, CompanyName) + None, CompanyName) # Truncate the current database. - print 'DROPPING current database...', + print('DROPPING current database...', end=' ') sys.stdout.flush() dropTables(DB_TABLES) - print 'DONE!' + print('DONE!') executeCustomQueries('BEFORE_CREATE') # Rebuild the database structure. - print 'CREATING new tables...', + print('CREATING new tables...', end=' ') sys.stdout.flush() createTables(DB_TABLES) - print 'DONE!' + print('DONE!') t('dropping and recreating the database') executeCustomQueries('AFTER_CREATE') @@ -2940,14 +3052,14 @@ def run(): readMovieList() # Comment readMovieList() and uncomment the following two lines # to keep the current info in the name and title tables. - ##CACHE_MID.populate() + # CACHE_MID.populate() t('readMovieList()') executeCustomQueries('BEFORE_COMPANIES') # distributors, miscellaneous-companies, production-companies, # special-effects-companies. - ##CACHE_COMPID.populate() + # CACHE_COMPID.populate() doMovieCompaniesInfo() # Do this now, and free some memory. CACHE_COMPID.flush() @@ -2959,8 +3071,8 @@ def run(): # costume-designers, directors, editors, miscellaneous, # production-designers. castLists() - ##CACHE_PID.populate() - ##CACHE_CID.populate() + # CACHE_PID.populate() + # CACHE_CID.populate() # Aka names and titles. doAkaNames() @@ -3014,7 +3126,7 @@ def run(): return if CSV_DIR: - print 'loading CSV files into the database' + print('loading CSV files into the database') executeCustomQueries('BEFORE_CSV_LOAD') loadCSVFiles() t('loadCSVFiles()') @@ -3032,41 +3144,39 @@ def run(): _HEARD = 0 + + def _kdb_handler(signum, frame): """Die gracefully.""" global _HEARD if _HEARD: - print "EHI! DON'T PUSH ME! I'VE HEARD YOU THE FIRST TIME! :-)" + print("EHI! DON'T PUSH ME! I'VE HEARD YOU THE FIRST TIME! :-)") return - print 'INTERRUPT REQUEST RECEIVED FROM USER. FLUSHING CACHES...' + print('INTERRUPT REQUEST RECEIVED FROM USER. FLUSHING CACHES...') _HEARD = 1 # XXX: trap _every_ error? - try: CACHE_MID.flush() - except IntegrityError: pass - try: CACHE_PID.flush() - except IntegrityError: pass - try: CACHE_CID.flush() - except IntegrityError: pass - try: CACHE_COMPID.flush() - except IntegrityError: pass - print 'DONE! (in %d minutes, %d seconds)' % \ - divmod(int(time.time())-BEGIN_TIME, 60) + try: + CACHE_MID.flush() + except IntegrityError: + pass + try: + CACHE_PID.flush() + except IntegrityError: + pass + try: + CACHE_CID.flush() + except IntegrityError: + pass + try: + CACHE_COMPID.flush() + except IntegrityError: + pass + print('DONE! (in %d minutes, %d seconds)' % + divmod(int(time.time()) - BEGIN_TIME, 60)) sys.exit() if __name__ == '__main__': - try: - print 'IMPORTING psyco...', - sys.stdout.flush() - #import DONOTIMPORTPSYCO - import psyco - #psyco.log() - psyco.profile() - print 'DONE!' - print '' - except ImportError: - print 'FAILED (not a big deal, everything is alright...)' - print '' import signal signal.signal(signal.SIGINT, _kdb_handler) if CSV_ONLY_LOAD: diff -pruN 5.1-1/bin/s32imdbpy.py 6.6-1/bin/s32imdbpy.py --- 5.1-1/bin/s32imdbpy.py 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/bin/s32imdbpy.py 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +s32imdbpy.py script. + +This script imports the s3 dataset distributed by IMDb into a SQL database. + +Copyright 2017-2018 Davide Alberani + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" + +import os +import glob +import gzip +import logging +import argparse +import sqlalchemy + +from imdb.parser.s3.utils import DB_TRANSFORM, title_soundex, name_soundexes + +TSV_EXT = '.tsv.gz' +# how many entries to write to the database at a time. +BLOCK_SIZE = 10000 + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +metadata = sqlalchemy.MetaData() + + +def generate_content(fd, headers, table): + """Generate blocks of rows to be written to the database. + + :param fd: a file descriptor for the .tsv.gz file + :type fd: :class:`_io.TextIOWrapper` + :param headers: headers in the file + :type headers: list + :param table: the table that will populated + :type table: :class:`sqlalchemy.Table` + :returns: block of data to insert + :rtype: list + """ + data = [] + headers_len = len(headers) + data_transf = {} + table_name = table.name + for column, conf in DB_TRANSFORM.get(table_name, {}).items(): + if 'transform' in conf: + data_transf[column] = conf['transform'] + for line in fd: + s_line = line.decode('utf-8').strip().split('\t') + if len(s_line) != headers_len: + continue + info = dict(zip(headers, [x if x != r'\N' else None for x in s_line])) + for key, tranf in data_transf.items(): + if key not in info: + continue + info[key] = tranf(info[key]) + if table_name == 'title_basics': + info['t_soundex'] = title_soundex(info['primaryTitle']) + elif table_name == 'title_akas': + info['t_soundex'] = title_soundex(info['title']) + elif table_name == 'name_basics': + info['ns_soundex'], info['sn_soundex'], info['s_soundex'] = name_soundexes(info['primaryName']) + data.append(info) + if len(data) >= BLOCK_SIZE: + yield data + data = [] + if data: + yield data + data = [] + + +def build_table(fn, headers): + """Build a Table object from a .tsv.gz file. + + :param fn: the .tsv.gz file + :type fn: str + :param headers: headers in the file + :type headers: list + """ + logging.debug('building table for file %s' % fn) + table_name = fn.replace(TSV_EXT, '').replace('.', '_') + table_map = DB_TRANSFORM.get(table_name) or {} + columns = [] + all_headers = set(headers) + all_headers.update(table_map.keys()) + for header in all_headers: + col_info = table_map.get(header) or {} + col_type = col_info.get('type') or sqlalchemy.UnicodeText + if 'length' in col_info and col_type is sqlalchemy.String: + col_type = sqlalchemy.String(length=col_info['length']) + col_args = { + 'name': header, + 'type_': col_type, + 'index': col_info.get('index', False) + } + col_obj = sqlalchemy.Column(**col_args) + columns.append(col_obj) + return sqlalchemy.Table(table_name, metadata, *columns) + + +def import_file(fn, engine): + """Import data from a .tsv.gz file. + + :param fn: the .tsv.gz file + :type fn: str + :param engine: SQLAlchemy engine + :type engine: :class:`sqlalchemy.engine.base.Engine` + """ + logging.info('begin processing file %s' % fn) + connection = engine.connect() + count = 0 + with gzip.GzipFile(fn, 'r') as gz_file: + headers = gz_file.readline().decode('utf-8').strip().split('\t') + logging.debug('headers of file %s: %s' % (fn, ','.join(headers))) + table = build_table(os.path.basename(fn), headers) + try: + table.drop() + logging.debug('table %s dropped' % table.name) + except: + pass + insert = table.insert() + metadata.create_all(tables=[table]) + try: + for block in generate_content(gz_file, headers, table): + try: + connection.execute(insert, block) + except Exception as e: + logging.error('error processing data: %d entries lost: %s' % (len(block), e)) + continue + count += len(block) + except Exception as e: + logging.error('error processing data on table %s: %s' % (table.name, e)) + logging.info('end processing file %s: %d entries' % (fn, count)) + + +def import_dir(dir_name, engine): + """Import data from a series of .tsv.gz files. + + :param dir_name: directory containing the .tsv.gz files + :type dir_name: str + :param engine: SQLAlchemy engine + :type engine: :class:`sqlalchemy.engine.base.Engine` + """ + for fn in glob.glob(os.path.join(dir_name, '*%s' % TSV_EXT)): + if not os.path.isfile(fn): + logging.debug('skipping file %s' % fn) + continue + import_file(fn, engine) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('tsv_files_dir') + parser.add_argument('db_uri') + parser.add_argument('--verbose', help='increase verbosity', action='store_true') + args = parser.parse_args() + dir_name = args.tsv_files_dir + db_uri = args.db_uri + if args.verbose: + logger.setLevel(logging.DEBUG) + engine = sqlalchemy.create_engine(db_uri, echo=False) + metadata.bind = engine + import_dir(dir_name, engine) + diff -pruN 5.1-1/bin/search_character.py 6.6-1/bin/search_character.py --- 5.1-1/bin/search_character.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/search_character.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ search_character.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "character name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "character name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,28 +28,25 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of character objects). results = i.search_character(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - name.encode(out_encoding, 'replace')) -print 'characterID\t: imdbID : name' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + name)) +print('characterID\t: imdbID : name') # Print the long imdb name for every character. for character in results: - outp = u'%s\t\t: %s : %s' % (character.characterID, i.get_imdbID(character), - character['long imdb name']) - print outp.encode(out_encoding, 'replace') - - + outp = '%s\t\t: %s : %s' % (character.characterID, + i.get_imdbID(character), + character['long imdb name']) + print(outp) diff -pruN 5.1-1/bin/search_company.py 6.6-1/bin/search_company.py --- 5.1-1/bin/search_company.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/search_company.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ search_company.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "company name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "company name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,28 +28,24 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of company objects). results = i.search_company(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - name.encode(out_encoding, 'replace')) -print 'companyID\t: imdbID : name' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + name)) +print('companyID\t: imdbID : name') # Print the long imdb name for every company. for company in results: - outp = u'%s\t\t: %s : %s' % (company.companyID, i.get_imdbID(company), - company['long imdb name']) - print outp.encode(out_encoding, 'replace') - - + outp = '%s\t\t: %s : %s' % (company.companyID, i.get_imdbID(company), + company['long imdb name']) + print(outp) diff -pruN 5.1-1/bin/search_keyword.py 6.6-1/bin/search_keyword.py --- 5.1-1/bin/search_keyword.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/search_keyword.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ search_keyword.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "keyword name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "keyword name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,27 +28,23 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of keyword strings). results = i.search_keyword(name, results=20) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - name.encode(out_encoding, 'replace')) -print ' : keyword' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + name)) +print(' : keyword') # Print every keyword. for idx, keyword in enumerate(results): - outp = u'%d: %s' % (idx+1, keyword) - print outp.encode(out_encoding, 'replace') - - + outp = '%d: %s' % (idx+1, keyword) + print(outp) diff -pruN 5.1-1/bin/search_movie.py 6.6-1/bin/search_movie.py --- 5.1-1/bin/search_movie.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/search_movie.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ search_movie.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "movie title"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "movie title"' % sys.argv[0]) sys.exit(2) title = sys.argv[1] @@ -27,28 +28,24 @@ title = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() -title = unicode(title, in_encoding, 'replace') try: # Do the search, and get the results (a list of Movie objects). results = i.search_movie(title) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - title.encode(out_encoding, 'replace')) -print 'movieID\t: imdbID : title' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + title)) +print('movieID\t: imdbID : title') # Print the long imdb title for every movie. for movie in results: - outp = u'%s\t: %s : %s' % (movie.movieID, i.get_imdbID(movie), - movie['long imdb title']) - print outp.encode(out_encoding, 'replace') - - + outp = '%s\t: %s : %s' % (movie.movieID, i.get_imdbID(movie), + movie['long imdb title']) + print(outp) diff -pruN 5.1-1/bin/search_person.py 6.6-1/bin/search_person.py --- 5.1-1/bin/search_person.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/bin/search_person.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- """ search_person.py @@ -13,13 +14,13 @@ import sys try: import imdb except ImportError: - print 'You bad boy! You need to install the IMDbPY package!' + print('You bad boy! You need to install the IMDbPY package!') sys.exit(1) if len(sys.argv) != 2: - print 'Only one argument is required:' - print ' %s "person name"' % sys.argv[0] + print('Only one argument is required:') + print(' %s "person name"' % sys.argv[0]) sys.exit(2) name = sys.argv[1] @@ -27,28 +28,24 @@ name = sys.argv[1] i = imdb.IMDb() -in_encoding = sys.stdin.encoding or sys.getdefaultencoding() out_encoding = sys.stdout.encoding or sys.getdefaultencoding() -name = unicode(name, in_encoding, 'replace') try: # Do the search, and get the results (a list of Person objects). results = i.search_person(name) -except imdb.IMDbError, e: - print "Probably you're not connected to Internet. Complete error report:" - print e +except imdb.IMDbError as e: + print("Probably you're not connected to Internet. Complete error report:") + print(e) sys.exit(3) # Print the results. -print ' %s result%s for "%s":' % (len(results), - ('', 's')[len(results) != 1], - name.encode(out_encoding, 'replace')) -print 'personID\t: imdbID : name' +print(' %s result%s for "%s":' % (len(results), + ('', 's')[len(results) != 1], + name)) +print('personID\t: imdbID : name') # Print the long imdb name for every person. for person in results: - outp = u'%s\t: %s : %s' % (person.personID, i.get_imdbID(person), - person['long imdb name']) - print outp.encode(out_encoding, 'replace') - - + outp = '%s\t: %s : %s' % (person.personID, i.get_imdbID(person), + person['long imdb name']) + print(outp) diff -pruN 5.1-1/debian/changelog 6.6-1/debian/changelog --- 5.1-1/debian/changelog 2016-11-24 08:57:31.000000000 +0000 +++ 6.6-1/debian/changelog 2018-09-09 14:19:35.000000000 +0000 @@ -1,3 +1,20 @@ +imdbpy (6.6-1) unstable; urgency=medium + + * New upstream release. IMDbPY is only Python 3 now: + - Remove python-imdbpy and add python3-imdbpy + - Adjust build-deps and deps + - Package is now arch:all + - Update patches, remove no_install_doc_or_stuff_in_etc and + add do_not_install_scripts. And continue not installing scripts + under usr/bin/ + * Update Vcs-* fields to point to salsa. + * Update S-V to 4.2.1, no changes required. + * Update to dh 11. + * Update debian/copyright. + * Update debian/watch. + + -- Ana Beatriz Guerrero Lopez Sun, 09 Sep 2018 16:19:35 +0200 + imdbpy (5.1-1) unstable; urgency=medium * New upstream release: diff -pruN 5.1-1/debian/compat 6.6-1/debian/compat --- 5.1-1/debian/compat 2016-11-23 21:30:38.000000000 +0000 +++ 6.6-1/debian/compat 2018-09-09 14:12:49.000000000 +0000 @@ -1 +1 @@ -9 +11 diff -pruN 5.1-1/debian/control 6.6-1/debian/control --- 5.1-1/debian/control 2016-11-24 08:22:09.000000000 +0000 +++ 6.6-1/debian/control 2018-09-09 14:12:49.000000000 +0000 @@ -2,21 +2,22 @@ Source: imdbpy Section: python Priority: optional Maintainer: Ana Beatriz Guerrero Lopez -Build-Depends: debhelper (>= 9), +Build-Depends: debhelper (>= 11), dh-python, - python-all-dev (>= 2.6.6-3~), - python-setuptools -Standards-Version: 3.9.8 + python3, + python3-setuptools, + python3-sphinx, +Standards-Version: 4.2.1 Homepage: http://imdbpy.sourceforge.net -Vcs-Git: https://anonscm.debian.org/git/users/ana/imdbpy.git -Vcs-Browser: https://anonscm.debian.org/git/users/ana/imdbpy.git +Vcs-Git: https://salsa.debian.org/ana/imdbpy.git +Vcs-Browser: https://salsa.debian.org/ana/imdbpy -Package: python-imdbpy -Architecture: any -Depends: python, ${misc:Depends}, ${python:Depends}, ${shlibs:Depends} -Recommends: python-lxml -Suggests: python-sqlalchemy, python-sqlobject -Description: Python package to access the IMDb's movie database +Package: python3-imdbpy +Architecture: all +Depends: python3, ${misc:Depends}, ${shlibs:Depends} +Recommends: python3-lxml +Suggests: python3-sqlalchemy, python3-sqlobject +Description: Python package to access the IMDb's movie database (Python 3) IMDbPY is a Python package useful to retrieve and manage the data of the IMDb movie database about both movies and people. It can be very easily used by programmers and developers to provide diff -pruN 5.1-1/debian/copyright 6.6-1/debian/copyright --- 5.1-1/debian/copyright 2016-11-24 08:19:48.000000000 +0000 +++ 6.6-1/debian/copyright 2018-09-09 14:12:53.000000000 +0000 @@ -5,8 +5,8 @@ It was downloaded from http://imdbpy.sou Upstream author: Davide Alberani -Copyright © 2004-2016 Davide Alberani - © 2008 H. Turgut Uyar +Copyright © 2004-2018 Davide Alberani + © 2008-2017 H. Turgut Uyar License: @@ -25,7 +25,7 @@ License: Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. On Debian systems, the complete text of the GNU General Public License -can be found in `/usr/share/common-licenses/GPL'. +can be found in `/usr/share/common-licenses/GPL-2'. AUTHOR'S DISCLAIMER @@ -38,11 +38,11 @@ and data included on the IMDb's site is content suppliers and protected by United States and international copyright laws. -Please, read the IMDb's conditions of use in their website: -- http://www.imdb.com/help/show_article?conditions -- http://www.imdb.com/help/show_leaf?usedatasoftware -- any other notice in the http://www.imdb.com/ site. +Please read the IMDb's conditions of use on their website: +- https://www.imdb.com/conditions +- https://www.imdb.com/licensing +- any other notice on the https://www.imdb.com/ site -The Debian packaging is © 2006-2016, Ana Beatriz Guerrero Lopez -and is licensed under the GPL, see `/usr/share/common-licenses/GPL'. +The Debian packaging is © 2006-2018, Ana Beatriz Guerrero Lopez +and is licensed under the GPLv2 or later, see `/usr/share/common-licenses/GPL-2'. diff -pruN 5.1-1/debian/patches/do_not_install_scripts 6.6-1/debian/patches/do_not_install_scripts --- 5.1-1/debian/patches/do_not_install_scripts 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/debian/patches/do_not_install_scripts 2018-09-09 14:12:36.000000000 +0000 @@ -0,0 +1,11 @@ +--- a/setup.py ++++ b/setup.py +@@ -103,7 +103,7 @@ + 'classifiers': [_f for _f in classifiers.split("\n") if _f], + 'url': home_page, + 'download_url': dwnl_url, +- 'scripts': scripts, ++# 'scripts': scripts, + 'data_files': data_files, + 'install_requires': ['SQLAlchemy', 'lxml'], + 'extras_require': { diff -pruN 5.1-1/debian/patches/no_install_doc_or_stuff_in_etc 6.6-1/debian/patches/no_install_doc_or_stuff_in_etc --- 5.1-1/debian/patches/no_install_doc_or_stuff_in_etc 2016-11-23 21:09:18.000000000 +0000 +++ 6.6-1/debian/patches/no_install_doc_or_stuff_in_etc 1970-01-01 00:00:00.000000000 +0000 @@ -1,23 +0,0 @@ ---- imdbpy-4.2.orig/setup.py -+++ imdbpy-4.2/setup.py -@@ -95,7 +95,7 @@ - # standard=False so that it's not installed if both --without-sqlobject - # and --without-sqlalchemy are specified. - featSQL = setuptools.dist.Feature('access to SQL databases', standard=False, -- remove='imdb.parser.sql', scripts=sqlScripts) -+ remove='imdb.parser.sql')#, scripts=sqlScripts) - - features = { - 'cutils': featCutils, -@@ -127,9 +127,9 @@ - 'url': home_page, - 'download_url': dwnl_url, - # Scripts. -- 'scripts': scripts, -+# 'scripts': scripts, - # Documentation files. -- 'data_files': data_files, -+# 'data_files': data_files, - # C extensions. - #'ext_modules': [cutils], - # Requirements. XXX: maybe we can use extras_require? diff -pruN 5.1-1/debian/patches/series 6.6-1/debian/patches/series --- 5.1-1/debian/patches/series 2016-11-23 21:09:18.000000000 +0000 +++ 6.6-1/debian/patches/series 2018-09-09 14:12:36.000000000 +0000 @@ -1 +1 @@ -no_install_doc_or_stuff_in_etc +do_not_install_scripts diff -pruN 5.1-1/debian/python3-imdbpy.examples 6.6-1/debian/python3-imdbpy.examples --- 5.1-1/debian/python3-imdbpy.examples 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/debian/python3-imdbpy.examples 2018-09-09 14:13:01.000000000 +0000 @@ -0,0 +1 @@ +bin/* diff -pruN 5.1-1/debian/python-imdbpy.docs 6.6-1/debian/python-imdbpy.docs --- 5.1-1/debian/python-imdbpy.docs 2016-11-24 08:19:48.000000000 +0000 +++ 6.6-1/debian/python-imdbpy.docs 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ -docs/AUTHOR.txt -docs/CONTRIBUTORS.txt -docs/CREDITS.txt -docs/DISCLAIMER.txt -docs/FAQS.txt -docs/README.adult -docs/README.companies -docs/README.currentRole -docs/README.devel -docs/README.info2xml -docs/README.keywords -docs/README.local -docs/README.locale -docs/README.logging -docs/README.mobile -docs/README.newparsers -docs/README.package -docs/README.redesign -docs/README.series -docs/README.sqldb -docs/README.txt -docs/README.unicode -docs/README.users -docs/TODO.txt -docs/goodies/ -docs/imdbpy*.dtd -docs/imdbpy.cfg -docs/imdbpyPowered.png -docs/imdbpyico.png -docs/imdbpyico.xpm -docs/imdbpyico16x16.ico -docs/imdbpyico32x32.ico -docs/imdbpywin.bmp diff -pruN 5.1-1/debian/python-imdbpy.examples 6.6-1/debian/python-imdbpy.examples --- 5.1-1/debian/python-imdbpy.examples 2016-11-24 08:19:48.000000000 +0000 +++ 6.6-1/debian/python-imdbpy.examples 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -bin/* diff -pruN 5.1-1/debian/README.Debian 6.6-1/debian/README.Debian --- 5.1-1/debian/README.Debian 2016-11-23 21:09:18.000000000 +0000 +++ 6.6-1/debian/README.Debian 1970-01-01 00:00:00.000000000 +0000 @@ -1,44 +0,0 @@ -IMDdPY for Debian ------------------ - -This package installs just the imdbpy python modules. The source code includes -some examples that upstream author made with the goal of be executables, but -since these scripts need a little setup about how to access to IMDb data, i -decided it was better not install it directly. -Please, read upstream's README.txt file to learn how to make this setup. -You can make all the necessary changes editing the py files placed at: - -/usr/share/doc/python2.X-imdbpy/examples/ - -And after this, make a symbolic link, for example: -ln -s /usr/share/doc/python-imdbpy/examples/get_first_movie.py /usr/bin/get_first_movie - - -Besides, if you want to use a local copy of the whole IMDb's database, -read the "README.local" file. Or if you want to insert the content of the plain text -data files into a SQL database, read the "README.sqldb" file. - -imdbpy2sql.py is somewhat different: it's not an example, but a script -required to put the plain text data files in the MySQL database. You can find -it gzippped in the 'example' directory. - - -- Ana Beatriz Guerrero Lopez , Sun, 2 Apr 2006 22:42:30 +0200 - - -config file ------------ - -The configuration file imdbpy.cfg is shipped at -/usr/share/doc/python-imdbpy/imdbpy.cfg. -For more information about this file, read the file itself. - - -- Ana Beatriz Guerrero Lopez , Sun, 16 Dec 2007 17:21:48 +0100 - -sql support ------------ -If you want to use the 'sql' data access system, you will need to install -SQLObject or SQLAlchemy. Packaged as python-sqlobject and python-sqlalchemy, -both suggested in Suggests. - - -- Ana Beatriz Guerrero Lopez , Thu, 11 Dec 2008 02:46:30 +0100 - diff -pruN 5.1-1/debian/rules 6.6-1/debian/rules --- 5.1-1/debian/rules 2016-11-24 08:19:27.000000000 +0000 +++ 6.6-1/debian/rules 2018-09-09 14:12:36.000000000 +0000 @@ -1,9 +1,6 @@ #!/usr/bin/make -f %: - dh $@ --with python2 - - -override_dh_python2: - dh_python2 --no-guessing-deps + dh $@ --with python3 --buildsystem=pybuild +override_dh_auto_test: diff -pruN 5.1-1/debian/source/include-binaries 6.6-1/debian/source/include-binaries --- 5.1-1/debian/source/include-binaries 2016-11-23 21:09:18.000000000 +0000 +++ 6.6-1/debian/source/include-binaries 2018-09-09 14:12:36.000000000 +0000 @@ -6,3 +6,4 @@ imdb/locale/es/LC_MESSAGES/imdbpy.mo imdb/locale/fr/LC_MESSAGES/imdbpy.mo imdb/locale/it/LC_MESSAGES/imdbpy.mo imdb/locale/tr/LC_MESSAGES/imdbpy.mo +imdb/locale/pt_BR/LC_MESSAGES/imdbpy.mo diff -pruN 5.1-1/debian/watch 6.6-1/debian/watch --- 5.1-1/debian/watch 2016-11-23 21:09:18.000000000 +0000 +++ 6.6-1/debian/watch 2018-09-09 14:13:06.000000000 +0000 @@ -1,7 +1,6 @@ -# Compulsory line, this is a version 3 file -version=3 +version=4 -# Uncomment to find new files on sourceforge, for debscripts >= 2.9 -http://sf.net/imdbpy/IMDbPY-(.*)\.tar\.gz +opts=filenamemangle=s/.+\/v?(\d\S+)\.tar\.gz/-$1\.tar\.gz/ \ + https://github.com/alberanid/imdbpy/tags .*/v?(\d\S+)\.tar\.gz diff -pruN 5.1-1/docs/AUTHOR.txt 6.6-1/docs/AUTHOR.txt --- 5.1-1/docs/AUTHOR.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/AUTHOR.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ - -NOTE: see also CONTRIBUTORS.txt for a list of developers of important -portions of the code and CREDITS.txt for a list of people who -contributed with hints, patches and so on. - - -AUTHOR: Davide Alberani - -e-mail: da@erlug.linux.it or alberanid@libero.it -IMDbPY page: http://imdbpy.sf.net/ -my homepage: http://erlug.linux.it/~da/ (Italian) -my cine-blog: http://cinemelma.blogspot.com/ (Italian) -Jabber ID: alberanid@jabber.linux.it (I'm on-line only from time to time) -ICQ UIN: 83641305 (nick 'Mad77') (I'm on-line only from time to time) -PGP KeyID: 0x465BFD47 (the key is available in my homepage and - through every keyserver) - -Sometimes (very seldom) you can find me in IRC on channels #python, -#linux-it and so on over irc.freenode.net with the nick 'Mad77'. - -Feel free to contact me for any ideas, bug reports and everything else. :-) - diff -pruN 5.1-1/docs/Changelog.rst 6.6-1/docs/Changelog.rst --- 5.1-1/docs/Changelog.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/Changelog.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,1589 @@ +Changelog +========= + +* What's new in release 6.6 "Stranger Things" (05 Aug 2018) + + [general] + + - #154: exclude docs and etc directories from packaging + - introduce 'https' as an alias for 'http' + - #151: the 'in' operator also considers key names + - #172: fix for ASCII keys in XML output + - #174: improve XML output + - #179: introduce Travis CI at https://travis-ci.org/alberanid/imdbpy + + [http] + + - #149: store person birth and death dates in ISO8601 format + - #166: fix birth and death dates without itemprop attributes + - #160: fix series seasons list + - #155 and #165: ignore certificate to prevent validation errors + - #156: fix tech parser + - #157: full-size headshot for persons + - #161: fix string/unicode conversion in Python 2.7 + - #173: raw akas and raw release dates fields + - #178: fix mini biography parser + + [s3] + + - #158: fetch and search AKAs + - update the goodies/download-from-s3 script to use the datasets.imdbws.com site + + +* What's new in release 6.5 "Poultrygeist: Night of the Chicken Dead" (15 Apr 2018) + + [general] + + - converted the documentation to Sphinx rst format + + [http] + + - fix title parser for in-production movies + - parsers are based on piculet + - improve collection of full-size cover images + + +* What's new in release 6.4 "Electric Dreams" (14 Mar 2018) + + [http] + + - remove obsolete parsers + - remove Character objects + - fix for search parsers + + +* What's new in release 6.3 "Altered Carbon" (27 Feb 2018) + + [general] + + - documentation updates + - introduced the 'imdbpy' CLI + - s3 accessSystem to access the new dataset from IMDb + + [http] + + - fixes for IMDb site redesign + - Person parser fixes + - users review parser + - improve external sites parser + - switch from akas.imdb.com domain to www.imdb.com + - fix for synopsis + - fix for tv series episodes + + [s3] + + - ability to import and access all the information + + +* What's new in release 6.2 "Justice League" (19 Nov 2017) + + [general] + + - introduce check for Python version + - SQLAlchemy can be disabled using --without-sqlalchemy + - fix #88: configuration file parser + - update documentation + + [http] + + - fixed ratings parser + - moved cookies from json to Python source + + +* What's new in release 6.1 + + - skipped version 6.1 due to a wrong release on pypi + + +* What's new in release 6.0 "Life is Strange" (12 Nov 2017) + + [general] + + - now IMDbPY is a Python 3 package + - simplified the code base: #61 + - remove dependencies: SQLObject, BeautifulSoup, C compiler + - introduced a tox testsuite + - fix various parsers + + +* What's new in release 5.1 "Westworld" (13 Nov 2016) + + [general] + + - fix for company names containing square brackets. + - fix XML output when imdb long name is missing. + - fixes #33: unable to use --without-sql + + [http] + + - fix birth/death dates parsing. + - fix top/bottom lists. + - Persons's resume page parser (courtesy of codynhat) + - fixes #29: split color info + - parser for "my rating" (you have to use your own cookies) + + [sql] + + - sound track list correctly identified. + - fixes #50: process splitted data in order + - fixes #53: parser for movie-links + + +* What's new in release 5.0 "House of Cards" (02 May 2014) + + [general] + + - Spanish, French, Arabic, Bulgarian and German translations. + - Introduced the list of French articles. + - fix for GAE. + - download_applydiffs.py script. + - fixed wrong handling of encoding in episode titles + - renamed README.utf8 to README.unicode + + [http] + + - fixed searches (again). + - search results are always in English. + - updated the cookies. + - support for obtaining metacritic score and URL. + - fixed goofs parser. + - fixed url for top250. + - fixes for biography page. + - fix for quotes. + - better charset identification. + - category and spoiler status for goofs. + - changed query separators from ; to &. + - fix for episodes of unknown seasons. + - new cookie. + + [mobile] + + - fixed searches. + + [sql] + + - fix for MSSQL + + +* What's new in release 4.9 "Iron Sky" (15 Jun 2012) + + [general] + + - urls used to access the IMDb site can be configured. + - helpers function to handle movie AKAs in various + languages (code by Alberto Malagoli). + - renamed the 'articles' module into 'linguistics'. + - introduced the 'reraiseExceptions' option, to re-raise + evey caught exception. + + [http] + + - fix for changed search parameters. + - introduced a 'timeout' parameter for connections to the web server. + - fix for business information. + - parser for the new style of episodes list. + - unicode searches handled as iso8859-1. + - fix for garbage in AKA titles. + + [sql] + + - vastly improved the store/restore of imdbIDs; now it should be faster + and more accurate. + - now the 'name' table contains a 'gender' field that can be 'm', 'f' or NULL. + - fix for nicknames. + - fix for missing titles in the crazy credits file. + - handled exceptions creating indexes, foreign keys and + executing custom queries. + - fixed creation on index for keywords. + - excluded {{SUSPENDED}} titles. + + +* What's new in release 4.8.2 "The Big Bang Theory" (02 Nov 2011) + + [general] + + - fixed install path of locales. + + [http] + + - removed debug code. + + +* What's new in release 4.8 "Super" (01 Nov 2011) + + [general] + + - fix for a problem managing exceptions with Python 2.4. + - converted old-style exceptions to instances. + - enanchements for the reduce.sh script. + - added notes about problems connecting to IMDb's web servers. + - improvements in the parsers of movie titles. + - improvements in the parser of person names. + + [http] + + - potential fix for GAE environment. + - handled the new style of "in production" information. + - fix for 'episodes' list. + - fix for 'episodes rating'. + - fix for queries that returned too many results. + - fix for wrong/missing references. + - removed no more available information set "amazon + reviews" and "dvd". + - fix for cast of tv series. + - fix for title of tv series. + - now the beautiful parses work again. + + [httpThin] + + - removed "httpThin", falling back to "http". + + [mobile] + + - fix for missing headshots. + - fix for rating and number of votes. + - fix for missing genres. + - many other fixes to keep up-to-date with the IMDb site. + + [sql] + + - fix for a nasty bug parsing notes about character names. + - fixes for SQLite with SQLOjbect. + + +* What's new in release 4.7 "Saw VI" (23 Jan 2011) + + [http] + + - first fixes for the new set of parsers. + - first changes to support the new set of web pages. + - fix for lists of uncategorized episodes. + - fix for movies with multiple countries. + - fix for the currentRole property. + - more robust handling for vote details. + + [mobile] + + - first fixes for the new set of parsers. + + [sql] + + - the tables containing titles and names (and akas) now + include a 'md5sum' column calculated on the "long imdb canonical title/name". + + +* What's new in release 4.6 "The Road" (19 Jun 2010) + + [general] + + - introduced the 'full-size cover url' and 'full-size headshot' + keys for Movie, Person and Character instances. + - moved the development to a Mercurial repository. + - introduced the parseXML function in the imdb.helpers module. + - now the asXML method can exclude dynamically generated keys. + - rationalized the use of the 'logging' and 'warnings' modules. + - the 'update' method no longer raises an exception, if asked for + an unknown info set. + + [http/mobile] + + - removed new garbage from the imdb pages. + - support new style of akas. + - fix for the "trivia" page. + - fixes for searches with too many results. + + [sql] + + - fixes for garbage in the plain text data files. + - support for SQLite shipped with Python 2.6. + + +* What's new in release 4.5.1 "Dollhouse" (01 Mar 2010) + + [general] + + - reintroduced the ez_setup.py file. + - fixes for AKAs on 'release dates'. + - added the dtd. + + +* What's new in release 4.5 "Invictus" (28 Feb 2010) + + [general] + + - moved to setuptools 0.6c11. + - trying to make the SVN release versions work fine. + - http/mobile should work in GAE (Google App Engine). + - added some goodies scripts, useful for programmers (see the + docs/goodies directory). + + [http/mobile] + + - removed urllib-based User-Agent header. + - fixes for some minor changes to IMDb's html. + - fixes for garbage in movie quotes. + - improvements in the handling of AKAs. + + [mobile] + + - fixes for AKAs in search results. + + [sql] + + - fixes for bugs restoring imdbIDs. + - first steps to split CSV creation/insertion. + + +* What's new in release 4.4 "Gandhi" (06 Jan 2010) + + [general] + + - introduced a logging facility; see README.logging. + - the 'http' and 'mobile' should be a lot more robust. + + [http] + + - fixes for the n-th set of changes to IMDb's HTML. + - improvements to perfect-match searches. + - slightly simplified the parsers for search results. + + [mobile] + + - fixes for the n-th set of changes to IMDb's HTML. + - slightly simplified the parsers for search results. + + [sql] + + - movies' keywords are now correctly imported, using CSV files. + - minor fixes to handle crap in the plain text data files. + - removed an outdate parameter passed to SQLObject. + - made imdbpy2sql.py more robust in some corner-cases. + - fixes for the Windows environment. + + +* What's new in release 4.3 "Public Enemies" (18 Nov 2009) + + [general] + + - the installer now takes care of .mo files. + - introduced, in the helpers module, the functions keyToXML and + translateKey, useful to translate dictionary keys. + - support for smart guessing of the language of a movie title. + - updated the DTD. + + [http] + + - fixed a lot of bugs introduced by the new IMDb.com design. + - nicer handling of HTTP 404 response code. + - fixed parsers for top250 and bottom100 lists. + - fixed a bug parsing AKAs. + - fixed misc bugs. + + [mobile] + + - removed duplicates in list of genres. + + [sql] + + - fixed a bug in the imdbpy2sql.py script using CSV files; + the 'movie_info_idx' and 'movie_keyword' were left + empty/with wrong data. + + +* What's new in release 4.2 "Battlestar Galactica" (31 Aug 2009) + + [general] + + - the 'local' data access system is gone. See README.local. + - the imdb.parser.common package was removed, and its code integrated + in imdb.parser.sql and in the imdbpy2sql.py script. + - fixes for the installer. + - the helpers module contains the fullSizeCoverURL function, to convert + a Movie, Person or Character instance (or a URL in a string) + in an URL to the full-size version of its cover/headshot. + Courtesy of Basil Shubin. + - used a newer version of msgfmt.py, to work around a hideous bug + generating locales. + - minor updates to locales. + - updated the DTD to version 4.2. + + [http] + + - removed garbage at the end of quotes. + - fixed problems parsing company names and notes. + - keys in character's quotes dictionary are now Movie instances. + - fixed a bug converting entities char references (affected BeautifulSoup). + - fixed a long-standing bug handling & with BeautifulSoup. + - top250 is now correctly parsed by BeautifulSoup. + + [sql] + + - fixed DB2 call for loading blobs/cblobs. + - information from obsolete files are now used if and only if they + refer to still existing titles. + - the --fix-old-style-titles argument is now obsolete. + + +* What's new in release 4.1 "State Of Play" (02 May 2009) + + [general] + + - DTD definition. + - support for locale. + - support for the new style for movie titles ("The Title" and no + more "Title, The" is internally used). + - minor fix to XML code to work with the test-suite. + + [http] + + - char references in the &#xHEXCODE; format are handled. + - fixed a bug with movies containing '....' in titles. And I'm + talking about Malcolm McDowell's filmography! + - 'airing' contains object (so the accessSystem variable is set). + - 'tv schedule' ('airing') pages of episodes can be parsed. + - 'tv schedule' is now a valid alias for 'airing'. + - minor fixes for empty/wrong strings. + + [sql] + + - in the database, soundex values for titles are always calculated + after the article is stripped (if any). + - imdbpy2sql.py has the --fix-old-style-titles option, to handle + files in the old format. + - fixed a bug saving imdbIDs. + + [local] + + - the 'local' data access system should be considered obsolete, and + will probably be removed in the next release. + + +* What's new in release 4.0 "Watchmen" (12 Mar 2009) + + [general] + + - the installer is now based on setuptools. + - new functions get_keyword and search_keyword to handle movie's keywords + (example scripts included). + - Movie/Person/... keys (and whole instances) can be converted to XML. + - two new functions, get_top250_movies and get_bottom100_movies, to + retrieve lists of best/worst movies (example scripts included). + - searching for movies and persons - if present - the 'akas' keyword + is filled, in the results. + - 'quotes' for movies is now always a list of lists. + - the old set of parsers (based on sgmllib.SGMLParser) are gone. + - fixed limitations handling multiple roles (with notes). + - fixed a bug converting somethingIDs to real imdbIDs. + - fixed some summary methods. + - updates to the documentation. + + [http] + + - adapted BeautifulSoup to lxml (internally, the lxml API is used). + - currentRole is no longer populated, for non-cast entries (everything + ends up into .notes). + - fixed a bug search for too common terms. + - fixed a bug identifying 'kind', searching for titles. + - fixed a bug parsing airing dates. + - fixed a bug searching for company names (when there's a direct hit). + - fixed a bug handling multiple characters. + - fixed a bug parsing episode ratings. + - nicer keys for technical details. + - removed the 'agent' page. + + [sql] + + - searching for a movie, the original titles are returned, instead + of AKAs. + - support for Foreign Keys. + - minor changes to the db's design. + - fixed a bug populating tables with SQLAlchemy. + - imdbpy2sql.py shows user time and system time, along with wall time. + + [local] + + - searching for a movie, the original titles are returned, instead + of AKAs. + + +* What's new in release 3.9 "The Strangers" (06 Jan 2009) + + [general] + + - introduced the search_episode method, to search for episodes' titles. + - movie['year'] is now an integer, and no more a string. + - fixed a bug parsing company names. + - introduced the helpers.makeTextNotes function, useful to pretty-print + strings in the 'TEXT::NOTE' format. + + [http] + + - fixed a bug regarding movies listed in the Bottom 100. + - fixed bugs about tv mini-series. + - fixed a bug about 'series cast' using BeautifulSoup. + + [sql] + + - fixes for DB2 (with SQLAlchemy). + - improved support for movies' aka titles (for series). + - made imdbpy2sql.py more robust, catching exceptions even when huge + amounts of data are skipped due to errors. + - introduced CSV support in the imdbpy2sql.py script. + + +* What's new in release 3.8 "Quattro Carogne a Malopasso" (03 Nov 2008) + + [http] + + - fixed search system for direct hits. + - fixed IDs so that they always are str and not unicode. + - fixed a bug about plot without authors. + - for pages about a single episode of a series, "Series Crew" are + now separated items. + - introduced the preprocess_dom method of the DOMParserBase class. + - handling rowspan for DOMHTMLAwardsParser is no more a special case. + - first changes to remove old parsers. + + [sql] + + - introduced support for SQLAlchemy. + + [mobile] + + - fixed multiple 'nick names'. + - added 'aspect ratio'. + - fixed a "direct hit" bug searching for people. + + [global] + + - fixed search_* example scripts. + - updated the documentation. + + +* What's new in release 3.7 "Burn After Reading" (22 Sep 2008) + + [http] + + - introduced a new set of parsers, active by default, based on DOM/XPath. + - old parsers fixed; 'news', 'genres', 'keywords', 'ratings', 'votes', + 'tech', 'taglines' and 'episodes'. + + [sql] + + - the pure python soundex function now behaves correctly. + + [general] + + - minor updates to the documentation, with an introduction to the + new set of parsers and notes for packagers. + + +* What's new in release 3.6 "RahXephon" (08 Jun 2008) + + [general] + + - support for company objects for every data access systems. + - introduced example scripts for companies. + - updated the documentation. + + [http and mobile] + + - changes to support the new HTML for "plot outline" and some lists + of values (languages, genres, ...) + - introduced the set_cookies method to set cookies for IMDb's account and + the del_cookies method to remove the use of cookies; in the imdbpy.cfg + configuration file, options "cookie_id" and "cookie_uu" can be set to + the appropriate values; if "cookie_id" is None, no cookies are sent. + - fixed parser for 'news' pages. + - fixed minor bug fetching movie/person/character references. + + [http] + + - fixed a search problem, while not using the IMDbPYweb's account. + - fixed bugs searching for characters. + + [mobile] + + - fixed minor bugs parsing search results. + + [sql] + + - fixed a bug handling movieIDs, when there are some + inconsistencies in the plain text data files. + + [local] + + - access to 'mpaa' and 'miscellaneous companies' information. + + +* What's new in release 3.5 "Blade Runner" (19 Apr 2008) + + [general] + + - first changes to work on Symbian mobile phones. + - now there is an imdb.available_access_systems() function, that can + be used to get a list of available data access systems. + - it's possible to pass 'results' as a parameter of the imdb.IMDb + function; it sets the number of results to return for queries. + - fixed summary() method in Movie and Person, to correctly handle + unicode chars. + - the helpers.makeObject2Txt function now supports recursion over + dictionaries. + - cutils.c MXLINELEN increased from 512 to 1024; some critical + strcpy replaced with strncpy. + - fixed configuration parser to be compatible with Python 2.2. + - updated list of articles and some stats in the comments. + - documentation updated. + + [sql] + + - fixed minor bugs in imdbpy2sql.py. + - restores imdbIDs for characters. + - now CharactersCache honors custom queries. + - the imdbpy2sql.py's --mysql-force-myisam command line option can be + used to force usage of MyISAM tables on InnoDB databases. + - added some warnings to the imdbpy2sql.py script. + + [local] + + - fixed a bug in the fall-back function used to scan movie titles, + when the cutils module is not available. + - mini biographies are cut up to 2**16-1 chars, to prevent troubles + with some MySQL servers. + - fixed bug in characters4local.py, dealing with some garbage in the files. + + +* What's new in release 3.4 "Flatliners" (16 Dec 2007) + + [general] + + - *** NOTE FOR PACKAGERS *** in the docs directory there is the + "imdbpy.cfg" configuration file, which should be installed in /etc + or equivalent directory; the setup.py script *doesn't* manage its + installation. + - introduced a global configuration file to set IMDbPY's parameters. + - supported characters using "sql" and "local" data access systems. + - fixed a bug retrieving characterID from a character's name. + + [http] + + - fixed a bug in "release dates" parser. + - fixed bugs in "episodes" parser. + - fixed bugs reading "series years". + - stricter definition for ParserBase._re_imdbIDmatch regular expression. + + [mobile] + + - fixed bugs reading "series years". + - fixed bugs reading characters' filmography. + + [sql] + + - support for characters. + + [local] + + - support for characters. + - introduced the characters4local.py script. + + +* What's new in release 3.3 "Heroes" (18 Nov 2007) + + [general] + + - first support for character pages; only for "http" and "mobile", so far. + - support for multiple characters. + - introduced an helper function to pretty-print objects. + - added README.currentRole. + - fixed minor bug in the __hash__ method of the _Container class. + - fixed changes to some key names for movies. + - introduced the search_character.py, get_character.py and + get_first_character.py example scripts. + + [http] + + - full support for character pages. + - fixed a bug retrieving some 'cover url'. + - fixed a bug with multi-paragraphs biographies. + - parsers are now instanced on demand. + - accessSystem and modFunct are correctly set for every Movie, Person + and Character object instanced. + + [mobile] + + - full support for character pages. + + [sql] + + - extended functionality of the custom queries support for the + imdbpy2sql.py script to circumvent a problem with MS SQLServer. + - introducted the "--mysql-innodb" and "--ms-sqlserver" shortcuts + for the imdbpy2sql.py script. + - introduced the "--sqlite-transactions" shortcut to activate + transaction using SQLite which, otherwise, would have horrible + performances. + - fixed a minor bug with top/bottom ratings, in the imdbpy2sql.py script. + + [local] + + - filtered out some crap in the "quotes" plain text data files, which + also affected sql, importing the data. + + +* What's new in release 3.2 "Videodrome" (25 Sep 2007) + + [global] + + - now there's an unique place where "akas.imdb.com" is set, in the + main module. + - introduced __version__ and VERSION in the main module. + - minor improvements to the documentation. + + [http] + + - updated the main movie parser to retrieve the recently modified + cast section. + - updated the crazy credits parser. + - fixed a bug retrieving 'cover url'. + + [mobile] + + - fixed a bug parsing people's filmography when only one duty + was listed. + - updated to retrieve series' creator. + + [sql] + + - added the ability to perform custom SQL queries at the command + line of the imdbpy2sql.py script. + - minor fixes for the imdbpy2sql.py script. + + +* What's new in release 3.1 "The Snake King" (18 Jul 2007) + + [global] + + - the IMDbPYweb account now returns a single item, when a search + returns only one "good enough" match (this is the IMDb's default). + - updated the documentation. + - updated list of contributors and developers. + + [http] + + - supported the new result page for searches. + - supported the 'synopsis' page. + - supported the 'parents guide' page. + - fixed a bug retrieving notes about a movie's connections. + - fixed a bug for python2.2 (s60 mobile phones). + - fixed a bug with 'Production Notes/Status'. + - fixed a bug parsing role/duty and notes (also for httpThin). + - fixed a bug retrieving user ratings. + - fixed a bug (un)setting the proxy. + - fixed 2 bugs in movie/person news. + - fixed a bug in movie faqs. + - fixed a bug in movie taglines. + - fixed a bug in movie quotes. + - fixed a bug in movie title, in "full cast and crew" page. + - fixed 2 bugs in persons' other works. + + [sql] + + - hypothetical fix for a unicode problem in the imdbpy2sql.py script. + - now the 'imdbID' fields in the Title and Name tables are restored, + updating from an older version. + - fixed a nasty bug handling utf-8 strings in the imdbpy2sql.py script. + + [mobile] + + - supported the new result page for searches. + - fixed a bug for python2.2 (s60 mobile phones). + - fixed a bug searching for persons with single match and no + messages in the board. + - fixed a bug parsing role/duty and notes. + + +* What's new in release 3.0 "Spider-Man 3" (03 May 2007) + + [global] + + - IMDbPY now works with the new IMDb's site design; a new account is + used to access data; this affect a lot of code, especially in the + 'http', 'httpThin' and 'mobile' data access systems. + - every returned string should now be unicode; dictionary keywords are + _not_ guaranteed to be unicode (but they are always 7bit strings). + - fixed a bug in the __contains__ method of the Movie class. + - fix in the analyze_title() function to handle malformed episode + numbers. + + [http] + + - introduced the _in_content instance variable for objects instances of + ParserBase, True when inside the
tag. + Opening and closing this pair of tags two methods, named _begin_content() + and _end_content() are called with no parameters (by default, they do + nothing). + - in the utils module there's the build_person function, useful to create + a Person instance from the tipical formats found in the IMDb's web site. + - an analogue build_movie function can be used to instance Movie objects. + - inverted the getRefs default - now if not otherwise set, it's False. + - added a parser for the "merchandising" ("for sale") page for persons. + - the 'rating' parser now collects also 'rating' and 'votes' data. + - the HTMLMovieParser class (for movies) was rewritten from zero. + - the HTMLMaindetailsParser class (for persons) was rewritten from zero. + - unified the "episode list" and "episodes cast" parsers. + - fixed a bug parsing locations, which resulted in missing information. + - locations_parser splitted from "tech" parser. + - "connections" parser now handles the recently introduced notes. + + [http parser conversion] + + - these parsers worked out-of-the-box; airing, eprating, alternateversions, + dvd, goofs, keywords, movie_awards, movie_faqs, person_awards, rec, + releasedates, search_movie, search_person, soundclips, soundtrack, trivia, + videoclips. + - these parsers were fixed; amazonrev, connections, episodes, crazycredits, + externalrev, misclinks, newsgrouprev, news, officialsites, otherworks, + photosites, plot, quotes, ratings, sales, taglines, tech, business, + literature, publicity, trivia, videoclips, maindetails, movie. + + [mobile] + + - fixed to work with the new design. + - a lot of code is now shared amongst 'http' and 'mobile'. + + [sql] + + - fixes for other bugs related to unicode support. + - minor changes to slightly improve performances. + + +* What's new in release 2.9 "Rodan! The Flying Monster" (21 Feb 2007) + + [global] + + - on 19 February IMDb has redesigned its site; this is the last + IMDbPY's release to parse the "old layout" pages; from now on, + the development will be geared to support the new web pages. + See the README.redesign file for more information. + - minor clean-ups and functions added to the helpers module. + + [http] + + - fixed some unicode-related problems searching for movie titles and + person names; also changed the queries used to search titles/names. + - fixed a bug parsing episodes for tv series. + - fixed a bug retrieving movieID for tv series, searching for titles. + + [mobile] + + - fixed a problem searching exact matches (movie titles only). + - fixed a bug with cast entries, after minor changes to the IMDb's + web site HTML. + + [local and sql] + + - fixed a bug parsing birth/death dates and notes. + + [sql] + + - (maybe) fixed another unicode-related bug fetching data from a + MySQL database. Maybe. Maybe. Maybe. + + +* What's new in release 2.8 "Apollo 13" (14 Dec 2006) + + [general] + + - fix for environments where sys.stdin was overridden by a custom object. + + [http data access system] + + - added support for the movies' "FAQ" page. + - now the "full credits" (aka "full cast and crew") page can be parsed; + it's mostly useful for tv series, because this page is complete while + "combined details" contains only partial data. + E.g. + + ia.update(tvSeries, 'full credits') + + - added support for the movies' "on television" (ia.update(movie, "airing")) + - fixed a bug with 'miscellaneous companies'. + - fixed a bug retrieving the list of episodes for tv series. + - fixed a bug with tv series episodes' cast. + - generic fix for XML single tags (unvalid HTML tags) like
+ - fixed a minor bug with 'original air date'. + + [sql data access system] + + - fix for a unicode bug with recent versions of SQLObject and MySQL. + - fix for a nasty bug in imdbpy2sql.py that will show up splitting a + data set too large to be sent in a single shot to the database. + + [mobile data access system] + + - fixed a bug searching titles and names, where XML char references + were not converted. + + +* What's new in release 2.7 "Pitch Black" (26 Sep 2006) + + [general] + + - fixed search_movie.py and search_person.py scripts; now they return + both the movieID/personID and the imdbID. + - the IMDbPY account was configured to hide the mini-headshots. + - http and mobile data access systems now try to handle queries + with too many results. + + [http data access system] + + - fixed a minor bug retrieving information about persons, with movies + in production. + - fixed support for cast list of tv series. + - fixed a bug retrieving 'plot keywords'. + - some left out company credits are now properly handled. + + [mobile data access system] + + - fixed a major bug with the cast list, after the changes to the + IMDb web site. + - fixed support for cast list of tv series. + - fixed a minor bug retrieving information about persons, with movies + in production. + - now every AKA title is correctly parsed. + + [sql data access system] + + - fixed a(nother) bug updating imdbID for movies and persons. + - fixed a bug retrieving personID, while handling names references. + + [local data access system] + + - "where now" information now correctly handles multiple lines (also + affecting the imdbpy2sql.py script). + + +* What's new in release 2.6 "They Live" (04 Jul 2006) + + [general] + + - renamed sortMovies to cmpMovies and sortPeople to cmpPeople; these + function are now used to compare Movie/Person objects. + The cmpMovies also handles tv series episodes. + + [http data access system] + + - now information about "episodes rating" are retrieved. + - fixed a bug retrieving runtimes and akas information. + - fixed an obscure bug trying an Exact Primary Title/Name search when + the provided title was wrong/incomplete. + - support for the new format of the "DVD details" page. + + [sql data access system] + + - now at insert-time the tables doesn't have indexes, which are + added later, resulting in a huge improvement of the performances + of the imdbpy2sql.py script. + - searching for tv series episodes now works. + - fixed a bug inserting information about top250 and bottom10 films rank. + - fixed a bug sorting movies in people's filmography. + - fixed a bug filtering out adult-only movies. + - removed unused ForeignKeys in the dbschema module. + - fixed a bug inserting data in databases that require a commit() call, + after a call to executemany(). + - fixed a bug inserting aka titles in database that checks for foreign + keys consistency. + - fixed an obscure bug splitting too huge data sets. + - MoviesCache and PersonsCache are now flushed few times. + - fixed a bug handling excessive recursion. + - improved the exceptions handling. + + +* What's new in release 2.5 "Ninja Thunderbolt" (15 May 2006) + + [general] + + - support for tv series episodes; see the README.series file. + - modified the DISCLAIMER.txt file to be compliant to the debian guidelines. + - fixed a bug in the get_first_movie.py script. + - Movie and Person instances are now hashable, so that they can be used + as dictionary keys. + - modified functions analyze_title and build_title to support tv episodes. + - use isinstance for type checking. + - minor updates to the documentation. + - the imdbID for Movie and Person instances is now searched if either + one of movieID/personID and title/name is provided. + - introduced the isSame() method for both Movie and Person classes, + useful to compare object by movieID/personID and accessSystem. + - __contains__() methods are now recursive. + - two new functions in the IMDbBase class, title2imdbID() and name2imdbID() + are used to get the imdbID, given a movie title or person name. + - two new functions in the helpers module, sortedSeasons() and + sortedEpisodes(), useful to manage lists/dictionaries of tv series + episodes. + - in the helpers module, the get_byURL() function can be used to retrieve + a Movie or Person object for the given URL. + - renamed the "ratober" C module to "cutils". + - added CONTRIBUTORS.txt file. + + [http data access system] + + - fixed a bug regarding currentRole for tv series. + - fixed a bug about the "merchandising links" page. + + [http and mobile data access systems] + + - fixed a bug retrieving cover url for tv (mini) series. + + [mobile data access system] + + - fixed a bug with tv series titles. + - retrieves the number of episodes for tv series. + + [local data access system] + + - new get_episodes function in the cutils/ratober C module. + - search functions (both C and pure python) are now a lot faster. + - updated the documentation with work-arounds to make the mkdb program + works with a recent set of plain text data files. + + [sql data access system] + + - uses the SQLObject ORM to support a wide range of database engines. + - added in the cutils C module the soundex() function, and a fall back + Python only version in the parser.sql package. + + +* What's new in release 2.4 "Munich" (09 Feb 2006) + + [general] + + - strings are now unicode/utf8. + - unified Movie and Person classes. + - the strings used to store every kind of information about movies and + person now are modified (substituting titles and names references) + only when it's really needed. + - speed improvements in functions modifyStrings, sortMovies, + canonicalName, analyze_name, analyze_title. + - performance improvements in every data access system. + - removed the deepcopy of the data, updating Movie and Person + information. + - moved the "ratober" C module in the imdb.parser.common package, + being used by both ""http" and "sql" data access systems. + - C functions in the "ratober" module are always case insensitive. + - the setup.py script contains a work-around to make installation + go on even if the "ratober" C module can't be compiled (displaying + a warning), since it's now optional. + - minor updates to documentation, to keep it in sync with changes + in the code. + - the new helpers.py module contains functions useful to write + IMDbPY-based programs. + - new doc file README.utf8, about unicode support. + + [http data access system] + + - the ParserBase class now inherits from sgmllib.SGMLParser, + instead of htmllib.HTMLParser, resulting in a little improvement + in parsing speed. + - fixed a bug in the parser for the "news" page for movies and + persons. + - removed special handlers for entity and chardefs in the HTMLMovieParser + class. + - fixed bugs related to non-ascii chars. + - fixed a bug retrieving the URL of the cover. + - fixed a nasty bug retrieving the title field. + - retrieve the 'merchandising links' page. + - support for the new "episodes cast" page for tv series. + - fixed a horrible bug retrieving guests information for tv series. + + [sql data access system] + + - fixed the imdbpy2sql.py script, to handle files with spurious lines. + - searches for names and titles are now much faster, if the + imdb.parser.common.ratober C module is compiled and installed. + - imdbpy2sql.py now works also on partial data (i.e. if you've not + downloaded every single plain text file). + - imdbpy2sql.py considers also a couple of files in the contrib directory. + - searching names and titles, only the first 5 chars returned from + the SOUNDEX() SQL function are compared. + - should works if the database is set to unicode/utf-8. + + [mobile data access system] + + - fixed bugs related to non-ascii chars. + - fixed a bug retrieving the URL of the cover. + - retrieve currentRole/notes also for tv guest appearances. + + [local data access system] + + - it can work even if the "ratober" C module is not compiled; + obviously the pure python substitute is painfully slow (a + warning is issued). + + +* What's new in release 2.3 "Big Fish" (03 Dec 2005) + + [general] + + - uniformed numerous keys for Movie and Person objects. + - 'birth name' is now always in canonical form, and 'nick names' + are always normalized; these changes also affect the sql data + access system. + + [http data access system] + + - removed the 'imdb mini-biography by' key; the name of the author + is now prepended to the 'mini biography' key. + - fixed an obscure bug using more than one access system (http in + conjunction with mobile or httpThin). + - fixed a bug in amazon reviews. + + [mobile data access system] + + - corrected some bugs retrieving filmography and cast list. + + [sql data access system] + + - remove 'birth name' and 'nick names' from the list of 'akas'. + - in the SQL database, 'crewmembers' is now 'miscellaneous crew'. + - fixed a bug retrieving "guests" for TV Series. + + +* What's new in release 2.2 "The Thing" (17 Oct 2005) + + [general] + + - now the Person class has a 'billingPos' instance variable used to + keep record of the position of the person in the list of credits (as + an example, "Laurence Fishburne" is billed in 2nd position in the + cast list for the "Matrix, The (1999)" movie. + - added two functions to the utils module, to sort respectively + movies (by year/title/imdbIndex) and persons (by billingPos/name/imdbIndex). + - every data access system support the 'adultSearch' argument and the + do_adult_search() method to exclude the adult movies from your searches. + By default, adult movies are always listed. + - renamed the scripts, appending the ".py" extension. + - added an "IMDbPY Powered" logo and a bitmap used by the Windows installer. + - now Person and Movie objects always convert name/title to the canonical + format (Title, The). + - minor changes to the functions used to convert to "canonical format" + names and titles; they should be faster and with better matches. + - 'title' is the first argument, instancing a Movie object (instead + of 'movieID'). + - 'name' is the first argument, instancing a Movie object (instead + of 'personID'). + + [http data access system] + + - retrieves the 'guest appearances' page for TV series. + - fixed a bug retrieving newsgroup reviews urls. + - fixed a bug managing non-breaking spaces (they're truly a damnation!) + - fixed a bug with mini TV Series in people's biographies. + - now keywords are in format 'bullet-time' and no more 'Bullet Time'. + + [mobile data access system] + + - fixed a bug with direct hits, searching for a person's name. + - fixed a bug with languages and countries. + + [local data access system] + + - now cast entries are correctly sorted. + - new search system; it should return better matches in less + time (searching people's name is still somewhat slow); it's + also possibile to search for "long imdb canonical title/name". + - fixed a bug retrieving information about a movie with the same + person listed more than one time in a given role/duty (e.g., the + same director for different episodes of a TV series). Now it + works fine and it should also be a bit faster. + - 'notable tv guest appearences' in biography is now a list of Movie + objects. + - writers are sorted in the right order. + + [sql data access system] + + - search results are now sorted in correct order; difflib is used to + calculate strings similarity. + - new search SQL query and comparison algorithm; it should return + much better matches. + - searches for only a surname now returns much better results. + - fixed a bug in the imdbpy2sql.py script; now movie quotes are correctly + managed. + - added another role, 'guests', for notable tv guest appearences. + - writers are sorted in the right order. + - put also the 'birth name' and the 'nick names' in the akanames table. + + +* What's new in release 2.1 "Madagascar" (30 Aug 2005) + + [general] + + - introduced the "sql data access system"; now you can transfer the + whole content of the plain text data files (distributed by IMDb) + into a SQL database (MySQL, so far). + - written a tool to insert the plain text data files in a SQL database. + - fixed a bug in items() and values() methods of Movie and Person + classes. + - unified portions of code shared between "local" and "sql". + + [http data access system] + + - fixed a bug in the search_movie() and search_person() methods. + - parse the "external reviews", "newsgroup reviews", "newsgroup reviews", + "misc links", "sound clips", "video clips", "amazon reviews", "news" and + "photo sites" pages for movies. + - parse the "news" page for persons. + - fixed a bug retrieving personID and movieID within namesRefs + and titlesRefs. + + [local data access system] + + - fixed a bug; 'producer' data where scanned two times. + - some tags were missing for the laserdisc entries. + + [mobile data access system] + + - fixed a bug retrieving cast information (sometimes introduced + with "Cast overview" and sometimes with "Credited cast"). + - fixed a bug in the search_movie() and search_person() methods. + + +* What's new in release 2.0 "Land Of The Dead" (16 Jul 2005) + + [general] + + - WARNING! Now, using http and mobile access methods, movie/person + searches will include by default adult movie titles/pornstar names. + You can still deactivate this feature by setting the adultSearch + argument to false, or calling the do_adult_search() method with + a false value. + - fixed a bug using the 'all' keyword of the 'update' method. + + [http data access system] + + - added the "recommendations" page. + - the 'notes' instance variable is now correctly used to store + miscellaneous information about people in non-cast roles, replacing + the 'currentRole' variable. + - the adultSearch initialization argument is by default true. + - you can supply the proxy to use with the 'proxy' initialization + argument. + - retrieve the "plot outline" information. + - fixed a bug in the BasicMovieParser class, due to changes in the + IMDb's html. + - the "rating details" parse information about the total number + of voters, arithmetic mean, median and so on. The values are + stored as integers and floats, and no more as strings. + - dictionary keys in soundtrack are lowercase. + - fixed a bug with empty 'location' information. + + [mobile data access system] + + - number of votes, rating and top 250 rank are now integers/floats. + - retrieve the "plot outline" information. + + [local data access system] + + - number of votes, rating and top 250 rank are now integers/floats. + + +* What's new in release 1.9 "Ed Wood" (02 May 2005) + + [general] + + - introduced the new "mobile" data access system, useful for + small systems. It should be from 2 to 20 times faster than "http" + or "httpThin". + - the "http", "httpThin" and "mobile" data access system can now + search for adult movies. See the README.adult file. + - now it should works again with python 2.0 and 2.1. + - fixed a bug affecting performances/download time. + - unified some keywords amongst differents data access systems. + + [http data access system] + + - fixed some bugs; now it retrieves names akas correctly. + + +* What's new in release 1.8 "Paths Of Glory" (24 Mar 2005) + + [general] + + - introduced a new data access system "httpThin", useful for + systems with limited bandwidth and CPU power, like PDA, + hand-held devices and mobile phones. + - the setup.py script can be configured to not compile/install + the local access system and the example scripts (useful for + hand-held devices); introduced setup.cfg and MANIFEST.in files. + - updated the list of articles used to manage movie titles. + - removed the all_info tuples from Movie and Person classes, + since the list of available info sets depends on the access + system. I've added two methods to the IMDbBase class, + get_movie_infoset() and get_person_infoset(). + - removed the IMDbNotAvailable exception. + - unified some code in methods get_movie(), get_person() and + update() in IMDbBase class. + - minor updates to the documentation; added a 46x46 PNG icon. + - documentation for small/mobile systems. + + [Movie class] + + - renamed the m['notes'] item of Movie objects to m['episodes']. + + [Person class] + + - the p.__contains__(m) method can be used to check if the p + Person has worked in the m Movie. + + [local data access system] + + - gather information about "laserdisc", "literature" and "business". + - fixed a bug in ratober.c; now the search_name() function + handles search strings already in the "Surname, Name" format. + - two new methods, get_lastMovieID() and get_lastPersonID(). + + [http data access system] + + - limit the number of results for the query; this will save a + lot of bandwidth. + - fixed a bug retrieving the number of episodes of tv series. + - now it retrieves movies information about "technical specifications", + "business data", "literature", "soundtrack", "dvd" and "locations". + - retrieves people information about "publicity" and "agent". + + +* What's new in release 1.7 "Saw" (04 Feb 2005) + + [general] + + - Person class has two new keys; 'canonical name' and + 'long imdb canonical name', like "Gibson, Mel" and + "Gibson, Mel (I)". + - now titles and names are always internally stored in the + canonical format. + - search_movie() and search_person() methods return the + "read" movieID or personID (handling aliases). + - Movie and Person objects have a 'notes' instance attribute, + used to specify comments about the role of a person in a movie. + The Movie class can also contain a ['notes'] item, used to + store information about the runtime; e.g. (26 episodes). + - fixed minor bugs in the IMDbBase, Person and Movie classes. + - some performance improvements. + + [http data access system] + + - fixed bugs retrieving the currentRole. + - try to handle unicode chars; return unicode strings when required. + - now the searches return also "popular titles" and + "popular names" from the new IMDb's search system. + + [local data access system] + + - information about movie connections are retrieved. + - support for multiple biographies. + - now it works with Python 2.2 or previous versions. + - fixed a minor glitch in the initialization of the ratober C module. + - fixed a pair buffer overflows. + - fixed some (very rare) infinite loops bugs. + - it raises IMDbDataAccessError for (most of) I/O errors. + + [Movie class] + - fixed a bug getting the "long imdb canonical title". + + +* What's new in release 1.6 "Ninja Commandments" (04 Jan 2005) + + [general] + + - now inside Movie and Person object, the text strings (biography, + movie plot, etc.) contain titles and names references, like + "_Movie, The (1999)_ (qv)" or "'A Person' (qv)"; these reference + are transformed at access time with a user defined function. + - introduced _get_real_movieID and _get_real_personID methods + in the IMDbBase class, to handle title/name aliases for the + local access system. + - split the _normalize_id method in _normalize_movieID + and _normalize_personID. + - fixed some bugs. + + [Movie class] + + - now you can access the 'canonical title' and + 'long imdb canonical title' attributes, to get the movie title + in the format "Movie Title, The". + + [local data access system] + + - title and name aliases now work correctly. + - now get_imdbMovieID and get_imdbPersonID methods should + work in almost every case. + - people's akas are handled. + + [http data access system] + + - now the BasicMovieParser class can correctly gather the imdbID. + + +* What's new in release 1.5 "The Incredibles" (23 Dec 2004) + + [local database] + + - support a local installation of the IMDb database! + WOW! Now you can download the plain text data files from + http://imdb.com/interfaces.html and access those + information through IMDbPY! + + [general] + + - movie titles and person names are "fully normalized"; + Not "Matrix, The (1999)", but "The Matrix (1999)"; + Not "Cruise, Tom" but "Tom Cruise". + - get_mop_infoSet() methods can now return a tuple with the + dictionary data and a list of information sets they provided. + + [http data access system] + + - support for the new search system (yes, another one...) + - a lot of small fixes to stay up-to-date with the html + of the IMDb web server. + - modified the personParser module so that it will no + more download both "filmoyear" and "maindetails" pages; + now only the latter is parsed. + - movie search now correctly reports the movie year and index. + - gather "locations" information about a movie. + - modified the HTMLAwardsParser class so that it doesn't list + empty entries. + + +* What's new in release 1.4 "The Village" (10 Nov 2004) + + [http data access system] + + - modified the personParser.HTMLMaindetailsParser class, + because IMDb has changed the img tag for the headshot. + - now 'archive footage' is handled correctly. + + [IMDb class] + + - fixed minor glitches (missing "self" parameter in a + couple of methods). + + [misc] + + - now distutils installs also the example scripts in ./bin/* + + +* What's new in release 1.3 "House of 1000 Corpses" (6 Jul 2004) + + [http data access system] + + - modified the BasicMovieParser and BasicPersonParser classes, + because IMDb has removed the "pageflicker" from the html pages. + + [general] + + - the test suite was moved outside the tgz package. + + +* What's new in release 1.2 "Kill Bill" (2 May 2004) + + [general] + + - now it retrieves almost every available information about movie + and people! + - introduced the concept of "data set", to retrieve different sets + of information about a movie/person (so that it's possibile to + fetch only the needed information). + - introduced a test suite, using the PyUnit (unittest) module. + - fixed a nasty typo; the analyze_title and build_title functions + now use the strings 'tv mini series' and 'tv series' for the 'kind' + key (previously the 'serie' word ws used). + - new design; removed the mix-in class and used a factory pattern; + imdb.IMDb is now a function, which returns an instance of a class, + subclass of imdb.IMDbBase. + - introduced the build_name(name_dict) function in the utils module, + which takes a dictionary and build a long imdb name. + - fixed bugs in the analyze_name function; now it correctly raise + an IMDbParserError exception for empty/all spaces strings. + - now the analyze_title function sets only the meaningful + information (i.e.: no 'kind' or 'year' key, if they're not set) + + [http data access system] + + - removed all non-greedy regular expressions. + - removed all regular expressions in the movieParser module; now + self.rawdata is no more used to search "strange" matches. + - introduced a ParserBase class, used as base class for the parsers. + - retrieve information about the production status (pre-production, + announced, in production, etc.) + - mpaa is now a string. + - now when an IMDbDataAccessError is raised it shows also the + used proxy. + - minor changes to improve performances in the handle_data method of + the HTMLMovieParser class. + - minor changes to achieve a major performances improvement in + the BasicPersonParser class in the searchPersonParse module. + + [Movie class] + + - fixed a bug in isSameTitle method, now the accessSystem is correctly + checked. + - fixed some typos. + + [Person class] + + - minor changes to the isSamePerson method (now it uses the build_name + function). + + +* What's new in release 1.1 "Gigli" (17 Apr 2004) + + [general] + + - added support for persons (search & retrieve information about people). + - removed the dataSets module. + - removed the MovieTitle and the SearchMovieResults classes; now information + about the title is stored directly in the Movie object and the search + methods return simple lists (of Movie or Person objects). + - removed the IMDbTitleError exception. + - added the analyze_name() function in the imdb.utils module, which + returns a dictionary with the 'name' and 'imdbIndex' keys from the + given long imdb name string. + + [http data access system] + + - http search uses the new search system. + - moved the plotParser module content inside the movieParser module. + - fixed a minor bug handling AKAs for movie titles. + + [IMDb class] + + - introduced the update(obj) method of the IMDb class, to update + the information of the given object (a Movie or Person instance). + - added the get_imdbURL(obj) method if the IMDb class, which returns + the URL of the main IMDb page for the given object (a Movie or Person). + - renamed the 'kind' parameter of the IMDb class to 'accessSystem'. + + [Movie class] + + - now __str__() returns only the short name; the summary() method + returns a pretty-printed string for the Movie object. + - persons are no more simple strings, but Person objects (the role/duty + is stored in the currentRole variable of the object). + - isSameTitle(obj) method to compare two Movie objects even when + not all information are gathered. + - new __contains__() method, to check is a given person was in a movie. + + [misc] + + - updated the documentation. + - corrected some syntax/grammar errors. + + +* What's new in release 1.0 "Equilibrium" (01 Apr 2004) + + [general] + + - first public release. + - retrieve data only from the web server. + - search only for movie titles. diff -pruN 5.1-1/docs/Changelog.txt 6.6-1/docs/Changelog.txt --- 5.1-1/docs/Changelog.txt 2016-11-13 11:35:44.000000000 +0000 +++ 6.6-1/docs/Changelog.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,1275 +0,0 @@ - Changelog for IMDbPY - ==================== - -* What's the new in release 5.1 "Westworld" (13 Nov 2016) - [general] - - fix for company names containing square brackets. - - fix XML output when imdb long name is missing. - - fixes #33: unable to use --without-sql - - [http] - - fix birth/death dates parsing. - - fix top/bottom lists. - - Persons's resume page parser (courtesy of codynhat) - - fixes #29: split color info - - parser for "my rating" (you have to use your own cookies) - - [sql] - - sound track list correctly identified. - - fixes #50: process splitted data in order - - fixes #53: parser for movie-links - - -* What's the new in release 5.0 "House of Cards" (02 May 2014) - [general] - - Spanish, French, Arabic, Bulgarian and German translations. - - Introduced the list of French articles. - - fix for GAE. - - download_applydiffs.py script. - - fixed wrong handling of encoding in episode titles - - renamed README.utf8 to README.unicode - - [http] - - fixed searches (again). - - search results are always in English. - - updated the cookies. - - support for obtaining metacritic score and URL. - - fixed goofs parser. - - fixed url for top250. - - fixes for biography page. - - fix for quotes. - - better charset identification. - - category and spoiler status for goofs. - - changed query separators from ; to &. - - fix for episodes of unknown seasons. - - new cookie. - - [mobile] - - fixed searches. - - [sql] - - fix for MSSQL - - -* What's the new in release 4.9 "Iron Sky" (15 Jun 2012) - [general] - - urls used to access the IMDb site can be configured. - - helpers function to handle movie AKAs in various - languages (code by Alberto Malagoli). - - renamed the 'articles' module into 'linguistics'. - - introduced the 'reraiseExceptions' option, to re-raise - evey caught exception. - - [http] - - fix for changed search parameters. - - introduced a 'timeout' parameter for connections to the web server. - - fix for business information. - - parser for the new style of episodes list. - - unicode searches handled as iso8859-1. - - fix for garbage in AKA titles. - - [sql] - - vastly improved the store/restore of imdbIDs; now it should be faster - and more accurate. - - now the 'name' table contains a 'gender' field that can be 'm', 'f' or NULL. - - fix for nicknames. - - fix for missing titles in the crazy credits file. - - handled exceptions creating indexes, foreign keys and - executing custom queries. - - fixed creation on index for keywords. - - excluded {{SUSPENDED}} titles. - - -* What's the new in release 4.8.2 "The Big Bang Theory" (02 Nov 2011) - [general] - - fixed install path of locales. - - [http] - - removed debug code. - - -* What's the new in release 4.8 "Super" (01 Nov 2011) - [general] - - fix for a problem managing exceptions with Python 2.4. - - converted old-style exceptions to instances. - - enanchements for the reduce.sh script. - - added notes about problems connecting to IMDb's web servers. - - improvements in the parsers of movie titles. - - improvements in the parser of person names. - - [http] - - potential fix for GAE environment. - - handled the new style of "in production" information. - - fix for 'episodes' list. - - fix for 'episodes rating'. - - fix for queries that returned too many results. - - fix for wrong/missing references. - - removed no more available information set "amazon - reviews" and "dvd". - - fix for cast of tv series. - - fix for title of tv series. - - now the beautiful parses work again. - - [httpThin] - - removed "httpThin", falling back to "http". - - [mobile] - - fix for missing headshots. - - fix for rating and number of votes. - - fix for missing genres. - - many other fixes to keep up-to-date with the IMDb site. - - [sql] - - fix for a nasty bug parsing notes about character names. - - fixes for SQLite with SQLOjbect. - - -* What's the new in release 4.7 "Saw VI" (23 Jan 2011) - [http] - - first fixes for the new set of parsers. - - first changes to support the new set of web pages. - - fix for lists of uncategorized episodes. - - fix for movies with multiple countries. - - fix for the currentRole property. - - more robust handling for vote details. - - [mobile] - - first fixes for the new set of parsers. - - [sql] - - the tables containing titles and names (and akas) now - include a 'md5sum' column calculated on the "long imdb canonical title/name". - - -* What's the new in release 4.6 "The Road" (19 Jun 2010) - [general] - - introduced the 'full-size cover url' and 'full-size headshot' - keys for Movie, Person and Character instances. - - moved the development to a Mercurial repository. - - introduced the parseXML function in the imdb.helpers module. - - now the asXML method can exclude dynamically generated keys. - - rationalized the use of the 'logging' and 'warnings' modules. - - the 'update' method no longer raises an exception, if asked for - an unknown info set. - - [http/mobile] - - removed new garbage from the imdb pages. - - support new style of akas. - - fix for the "trivia" page. - - fixes for searches with too many results. - - [sql] - - fixes for garbage in the plain text data files. - - support for SQLite shipped with Python 2.6. - - -* What's the new in release 4.5.1 "Dollhouse" (01 Mar 2010) - [general] - - reintroduced the ez_setup.py file. - - fixes for AKAs on 'release dates'. - - added the dtd. - - -* What's the new in release 4.5 "Invictus" (28 Feb 2010) - [general] - - moved to setuptools 0.6c11. - - trying to make the SVN release versions work fine. - - http/mobile should work in GAE (Google App Engine). - - added some goodies scripts, useful for programmers (see the - docs/goodies directory). - - [http/mobile] - - removed urllib-based User-Agent header. - - fixes for some minor changes to IMDb's html. - - fixes for garbage in movie quotes. - - improvements in the handling of AKAs. - - [mobile] - - fixe for AKAs in search results. - - [sql] - - fixes for bugs restoring imdbIDs. - - first steps to split CSV creation/insertion. - - -* What's the new in release 4.4 "Gandhi" (06 Jan 2010) - [general] - - introduced a logging facility; see README.logging. - - the 'http' and 'mobile' should be a lot more robust. - - [http] - - fixes for the n-th set of changes to IMDb's HTML. - - improvements to perfect-match searches. - - slightly simplified the parsers for search results. - - [mobile] - - fixes for the n-th set of changes to IMDb's HTML. - - slightly simplified the parsers for search results. - - [sql] - - movies' keywords are now correctly imported, using CSV files. - - minor fixes to handle crap in the plain text data files. - - removed an outdate parameter passed to SQLObject. - - made imdbpy2sql.py more robust in some corner-cases. - - fixes for the Windows environment. - - -* What's the new in release 4.3 "Public Enemies" (18 Nov 2009) - [general] - - the installer now takes care of .mo files. - - introduced, in the helpers module, the functions keyToXML and - translateKey, useful to translate dictionary keys. - - support for smart guessing of the language of a movie title. - - updated the DTD. - - [http] - - fixed a lot of bugs introduced by the new IMDb.com design. - - nicer handling of HTTP 404 response code. - - fixed parsers for top250 and bottom100 lists. - - fixed a bug parsing AKAs. - - fixed misc bugs. - - [mobile] - - removed duplicates in list of genres. - - [sql] - - fixed a bug in the imdbpy2sql.py script using CSV files; - the 'movie_info_idx' and 'movie_keyword' were left - empty/with wrong data. - - -* What's the new in release 4.2 "Battlestar Galactica" (31 Aug 2009) - [general] - - the 'local' data access system is gone. See README.local. - - the imdb.parser.common package was removed, and its code integrated - in imdb.parser.sql and in the imdbpy2sql.py script. - - fixes for the installer. - - the helpers module contains the fullSizeCoverURL function, to convert - a Movie, Person or Character instance (or a URL in a string) - in an URL to the full-size version of its cover/headshot. - Courtesy of Basil Shubin. - - used a newer version of msgfmt.py, to work around a hideous bug - generating locales. - - minor updates to locales. - - updated the DTD to version 4.2. - - [http] - - removed garbage at the end of quotes. - - fixed problems parsing company names and notes. - - keys in character's quotes dictionary are now Movie instances. - - fixed a bug converting entities char references (affected BeautifulSoup). - - fixed a long-standing bug handling & with BeautifulSoup. - - top250 is now correctly parsed by BeautifulSoup. - - [sql] - - fixed DB2 call for loading blobs/cblobs. - - information from obsolete files are now used if and only if they - refer to still existing titles. - - the --fix-old-style-titles argument is now obsolete. - - -* What's the new in release 4.1 "State Of Play" (02 May 2009) - [general] - - DTD definition. - - support for locale. - - support for the new style for movie titles ("The Title" and no - more "Title, The" is internally used). - - minor fix to XML code to work with the test-suite. - - [http] - - char references in the &#xHEXCODE; format are handled. - - fixed a bug with movies containing '....' in titles. And I'm - talking about Malcolm McDowell's filmography! - - 'airing' contains object (so the accessSystem variable is set). - - 'tv schedule' ('airing') pages of episodes can be parsed. - - 'tv schedule' is now a valid alias for 'airing'. - - minor fixes for empty/wrong strings. - - [sql] - - in the database, soundex values for titles are always calculated - after the article is stripped (if any). - - imdbpy2sql.py has the --fix-old-style-titles option, to handle - files in the old format. - - fixed a bug saving imdbIDs. - - [local] - - the 'local' data access system should be considered obsolete, and - will probably be removed in the next release. - - -* What's the new in release 4.0 "Watchmen" (12 Mar 2009) - [general] - - the installer is now based on setuptools. - - new functions get_keyword and search_keyword to handle movie's keywords - (example scripts included). - - Movie/Person/... keys (and whole instances) can be converted to XML. - - two new functions, get_top250_movies and get_bottom100_movies, to - retrieve lists of best/worst movies (example scripts included). - - searching for movies and persons - if present - the 'akas' keyword - is filled, in the results. - - 'quotes' for movies is now always a list of lists. - - the old set of parsers (based on sgmllib.SGMLParser) are gone. - - fixed limitations handling multiple roles (with notes). - - fixed a bug converting somethingIDs to real imdbIDs. - - fixed some summary methods. - - updates to the documentation. - - [http] - - adapted BeautifulSoup to lxml (internally, the lxml API is used). - - currentRole is no longer populated, for non-cast entries (everything - ends up into .notes). - - fixed a bug search for too common terms. - - fixed a bug identifying 'kind', searching for titles. - - fixed a bug parsing airing dates. - - fixed a bug searching for company names (when there's a direct hit). - - fixed a bug handling multiple characters. - - fixed a bug parsing episode ratings. - - nicer keys for technical details. - - removed the 'agent' page. - - [sql] - - searching for a movie, the original titles are returned, instead - of AKAs. - - support for Foreign Keys. - - minor changes to the db's design. - - fixed a bug populating tables with SQLAlchemy. - - imdbpy2sql.py shows user time and system time, along with wall time. - - [local] - - searching for a movie, the original titles are returned, instead - of AKAs. - - -* What's the new in release 3.9 "The Strangers" (06 Jan 2009) - [general] - - introduced the search_episode method, to search for episodes' titles. - - movie['year'] is now an integer, and no more a string. - - fixed a bug parsing company names. - - introduced the helpers.makeTextNotes function, useful to pretty-print - strings in the 'TEXT::NOTE' format. - - [http] - - fixed a bug regarding movies listed in the Bottom 100. - - fixed bugs about tv mini-series. - - fixed a bug about 'series cast' using BeautifulSoup. - - [sql] - - fixes for DB2 (with SQLAlchemy). - - improved support for movies' aka titles (for series). - - made imdbpy2sql.py more robust, catching exceptions even when huge - amounts of data are skipped due to errors. - - introduced CSV support in the imdbpy2sql.py script. - - -* What's the new in release 3.8 "Quattro Carogne a Malopasso" (03 Nov 2008) - [http] - - fixed search system for direct hits. - - fixed IDs so that they always are str and not unicode. - - fixed a bug about plot without authors. - - for pages about a single episode of a series, "Series Crew" are - now separated items. - - introduced the preprocess_dom method of the DOMParserBase class. - - handling rowspan for DOMHTMLAwardsParser is no more a special case. - - first changes to remove old parsers. - - [sql] - - introduced support for SQLAlchemy. - - [mobile] - - fixed multiple 'nick names'. - - added 'aspect ratio'. - - fixed a "direct hit" bug searching for people. - - [global] - - fixed search_* example scripts. - - updated the documentation. - - -* What's the new in release 3.7 "Burn After Reading" (22 Sep 2008) - [http] - - introduced a new set of parsers, active by default, based on DOM/XPath. - - old parsers fixed; 'news', 'genres', 'keywords', 'ratings', 'votes', - 'tech', 'taglines' and 'episodes'. - - [sql] - - the pure python soundex function now behaves correctly. - - [general] - - minor updates to the documentation, with an introduction to the - new set of parsers and notes for packagers. - - -* What's the new in release 3.6 "RahXephon" (08 Jun 2008) - [general] - - support for company objects for every data access systems. - - introduced example scripts for companies. - - updated the documentation. - - [http and mobile] - - changes to support the new HTML for "plot outline" and some lists - of values (languages, genres, ...) - - introduced the set_cookies method to set cookies for IMDb's account and - the del_cookies method to remove the use of cookies; in the imdbpy.cfg - configuration file, options "cookie_id" and "cookie_uu" can be set to - the appropriate values; if "cookie_id" is None, no cookies are sent. - - fixed parser for 'news' pages. - - fixed minor bug fetching movie/person/character references. - - [http] - - fixed a search problem, while not using the IMDbPYweb's account. - - fixed bugs searching for characters. - - [mobile] - - fixed minor bugs parsing search results. - - [sql] - - fixed a bug handling movieIDs, when there are some - inconsistencies in the plain text data files. - - [local] - - access to 'mpaa' and 'miscellaneous companies' information. - - -* What's the new in release 3.5 "Blade Runner" (19 Apr 2008) - [general] - - first changes to work on Symbian mobile phones. - - now there is an imdb.available_access_systems() function, that can - be used to get a list of available data access systems. - - it's possible to pass 'results' as a parameter of the imdb.IMDb - function; it sets the number of results to return for queries. - - fixed summary() method in Movie and Person, to correctly handle - unicode chars. - - the helpers.makeObject2Txt function now supports recursion over - dictionaries. - - cutils.c MXLINELEN increased from 512 to 1024; some critical - strcpy replaced with strncpy. - - fixed configuration parser to be compatible with Python 2.2. - - updated list of articles and some stats in the comments. - - documentation updated. - - [sql] - - fixed minor bugs in imdbpy2sql.py. - - restores imdbIDs for characters. - - now CharactersCache honors custom queries. - - the imdbpy2sql.py's --mysql-force-myisam command line option can be - used to force usage of MyISAM tables on InnoDB databases. - - added some warnings to the imdbpy2sql.py script. - - [local] - - fixed a bug in the fall-back function used to scan movie titles, - when the cutils module is not available. - - mini biographies are cut up to 2**16-1 chars, to prevent troubles - with some MySQL servers. - - fixed bug in characters4local.py, dealing with some garbage in the files. - - -* What's the new in release 3.4 "Flatliners" (16 Dec 2007) - [general] - - *** NOTE FOR PACKAGERS *** in the docs directory there is the - "imdbpy.cfg" configuration file, which should be installed in /etc - or equivalent directory; the setup.py script _doesn't_ manage its - installation. - - introduced a global configuration file to set IMDbPY's parameters. - - supported characters using "sql" and "local" data access systems. - - fixed a bug retrieving characterID from a character's name. - - [http] - - fixed a bug in "release dates" parser. - - fixed bugs in "episodes" parser. - - fixed bugs reading "series years". - - stricter definition for ParserBase._re_imdbIDmatch regular expression. - - [mobile] - - fixed bugs reading "series years". - - fixed bugs reading characters' filmography. - - [sql] - - support for characters. - - [local] - - support for characters. - - introduced the characters4local.py script. - - -* What's the new in release 3.3 "Heroes" (18 Nov 2007) - [general] - - first support for character pages; only for "http" and "mobile", so far. - - support for multiple characters. - - introduced an helper function to pretty-print objects. - - added README.currentRole. - - fixed minor bug in the __hash__ method of the _Container class. - - fixed changes to some key names for movies. - - introduced the search_character.py, get_character.py and - get_first_character.py example scripts. - - [http] - - full support for character pages. - - fixed a bug retrieving some 'cover url'. - - fixed a bug with multi-paragraphs biographies. - - parsers are now instanced on demand. - - accessSystem and modFunct are correctly set for every Movie, Person - and Character object instanced. - - [mobile] - - full support for character pages. - - [sql] - - extended functionality of the custom queries support for the - imdbpy2sql.py script to circumvent a problem with MS SQLServer. - - introducted the "--mysql-innodb" and "--ms-sqlserver" shortcuts - for the imdbpy2sql.py script. - - introduced the "--sqlite-transactions" shortcut to activate - transaction using SQLite which, otherwise, would have horrible - performances. - - fixed a minor bug with top/bottom ratings, in the imdbpy2sql.py script. - - [local] - - filtered out some crap in the "quotes" plain text data files, which - also affected sql, importing the data. - - -* What's the new in release 3.2 "Videodrome" (25 Sep 2007) - [global] - - now there's an unique place where "akas.imdb.com" is set, in the - main module. - - introduced __version__ and VERSION in the main module. - - minor improvements to the documentation. - - [http] - - updated the main movie parser to retrieve the recently modified - cast section. - - updated the crazy credits parser. - - fixed a bug retrieving 'cover url'. - - [mobile] - - fixed a bug parsing people's filmography when only one duty - was listed. - - updated to retrieve series' creator. - - [sql] - - added the ability to perform custom SQL queries at the command - line of the imdbpy2sql.py script. - - minor fixes for the imdbpy2sql.py script. - - -* What's the new in release 3.1 "The Snake King" (18 Jul 2007) - [global] - - the IMDbPYweb account now returns a single item, when a search - returns only one "good enough" match (this is the IMDb's default). - - updated the documentation. - - updated list of contributors and developers. - - [http] - - supported the new result page for searches. - - supported the 'synopsis' page. - - supported the 'parents guide' page. - - fixed a bug retrieving notes about a movie's connections. - - fixed a bug for python2.2 (s60 mobile phones). - - fixed a bug with 'Production Notes/Status'. - - fixed a bug parsing role/duty and notes (also for httpThin). - - fixed a bug retrieving user ratings. - - fixed a bug (un)setting the proxy. - - fixed 2 bugs in movie/person news. - - fixed a bug in movie faqs. - - fixed a bug in movie taglines. - - fixed a bug in movie quotes. - - fixed a bug in movie title, in "full cast and crew" page. - - fixed 2 bugs in persons' other works. - - [sql] - - hypothetical fix for a unicode problem in the imdbpy2sql.py script. - - now the 'imdbID' fields in the Title and Name tables are restored, - updating from an older version. - - fixed a nasty bug handling utf-8 strings in the imdbpy2sql.py script. - - [mobile] - - supported the new result page for searches. - - fixed a bug for python2.2 (s60 mobile phones). - - fixed a bug searching for persons with single match and no - messages in the board. - - fixed a bug parsing role/duty and notes. - - -* What's the new in release 3.0 "Spider-Man 3" (03 May 2007) - [global] - - IMDbPY now works with the new IMDb's site design; a new account is - used to access data; this affect a lot of code, especially in the - 'http', 'httpThin' and 'mobile' data access systems. - - every returned string should now be unicode; dictionary keywords are - _not_ guaranteed to be unicode (but they are always 7bit strings). - - fixed a bug in the __contains__ method of the Movie class. - - fix in the analyze_title() function to handle malformed episode - numbers. - - [http] - - introduced the _in_content instance variable for objects instances of - ParserBase, True when inside the
tag. - Opening and closing this pair of tags two methods, named _begin_content() - and _end_content() are called with no parameters (by default, they do - nothing). - - in the utils module there's the build_person function, useful to create - a Person instance from the tipical formats found in the IMDb's web site. - - an analogue build_movie function can be used to instance Movie objects. - - inverted the getRefs default - now if not otherwise set, it's False. - - added a parser for the "merchandising" ("for sale") page for persons. - - the 'rating' parser now collects also 'rating' and 'votes' data. - - the HTMLMovieParser class (for movies) was rewritten from zero. - - the HTMLMaindetailsParser class (for persons) was rewritten from zero. - - unified the "episode list" and "episodes cast" parsers. - - fixed a bug parsing locations, which resulted in missing information. - - locations_parser splitted from "tech" parser. - - "connections" parser now handles the recently introduced notes. - - [http parser conversion] - - these parsers worked out-of-the-box; airing, eprating, alternateversions, - dvd, goofs, keywords, movie_awards, movie_faqs, person_awards, rec, - releasedates, search_movie, search_person, soundclips, soundtrack, trivia, - videoclips. - - these parsers were fixed; amazonrev, connections, episodes, crazycredits, - externalrev, misclinks, newsgrouprev, news, officialsites, otherworks, - photosites, plot, quotes, ratings, sales, taglines, tech, business, - literature, publicity, trivia, videoclips, maindetails, movie. - - [mobile] - - fixed to work with the new design. - - a lot of code is now shared amongst 'http' and 'mobile'. - - [sql] - - fixes for other bugs related to unicode support. - - minor changes to slightly improve performances. - - -* What's the new in release 2.9 "Rodan! The Flying Monster" (21 Feb 2007) - [global] - - on 19 February IMDb has redesigned its site; this is the last - IMDbPY's release to parse the "old layout" pages; from now on, - the development will be geared to support the new web pages. - See the README.redesign file for more information. - - minor clean-ups and functions added to the helpers module. - - [http] - - fixed some unicode-related problems searching for movie titles and - person names; also changed the queries used to search titles/names. - - fixed a bug parsing episodes for tv series. - - fixed a bug retrieving movieID for tv series, searching for titles. - - [mobile] - - fixed a problem searching exact matches (movie titles only). - - fixed a bug with cast entries, after minor changes to the IMDb's - web site HTML. - - [local and sql] - - fixed a bug parsing birth/death dates and notes. - - [sql] - - (maybe) fixed another unicode-related bug fetching data from a - MySQL database. Maybe. Maybe. Maybe. - - -* What's the new in release 2.8 "Apollo 13" (14 Dec 2006) - [general] - - fix for environments where sys.stdin was overridden by a custom object. - - [http data access system] - - added support for the movies' "FAQ" page. - - now the "full credits" (aka "full cast and crew") page can be parsed; - it's mostly useful for tv series, because this page is complete while - "combined details" contains only partial data. - E.g. - ia.update(tvSeries, 'full credits') - - added support for the movies' "on television" (ia.update(movie, "airing")) - - fixed a bug with 'miscellaneous companies'. - - fixed a bug retrieving the list of episodes for tv series. - - fixed a bug with tv series episodes' cast. - - generic fix for XML single tags (unvalid HTML tags) like
- - fixed a minor bug with 'original air date'. - - [sql data access system] - - fix for a unicode bug with recent versions of SQLObject and MySQL. - - fix for a nasty bug in imdbpy2sql.py that will show up splitting a - data set too large to be sent in a single shot to the database. - - [mobile data access system] - - fixed a bug searching titles and names, where XML char references - were not converted. - - -* What's the new in release 2.7 "Pitch Black" (26 Sep 2006) - [general] - - fixed search_movie.py and search_person.py scripts; now they return - both the movieID/personID and the imdbID. - - the IMDbPY account was configured to hide the mini-headshots. - - http and mobile data access systems now try to handle queries - with too many results. - - [http data access system] - - fixed a minor bug retrieving information about persons, with movies - in production. - - fixed support for cast list of tv series. - - fixed a bug retrieving 'plot keywords'. - - some left out company credits are now properly handled. - - [mobile data access system] - - fixed a major bug with the cast list, after the changes to the - IMDb web site. - - fixed support for cast list of tv series. - - fixed a minor bug retrieving information about persons, with movies - in production. - - now every AKA title is correctly parsed. - - [sql data access system] - - fixed a(nother) bug updating imdbID for movies and persons. - - fixed a bug retrieving personID, while handling names references. - - [local data access system] - - "where now" information now correctly handles multiple lines (also - affecting the imdbpy2sql.py script). - - -* What's the new in release 2.6 "They Live" (04 Jul 2006) - [general] - - renamed sortMovies to cmpMovies and sortPeople to cmpPeople; these - function are now used to compare Movie/Person objects. - The cmpMovies also handles tv series episodes. - - [http data access system] - - now information about "episodes rating" are retrieved. - - fixed a bug retrieving runtimes and akas information. - - fixed an obscure bug trying an Exact Primary Title/Name search when - the provided title was wrong/incomplete. - - support for the new format of the "DVD details" page. - - [sql data access system] - - now at insert-time the tables doesn't have indexes, which are - added later, resulting in a huge improvement of the performances - of the imdbpy2sql.py script. - - searching for tv series episodes now works. - - fixed a bug inserting information about top250 and bottom10 films rank. - - fixed a bug sorting movies in people's filmography. - - fixed a bug filtering out adult-only movies. - - removed unused ForeignKeys in the dbschema module. - - fixed a bug inserting data in databases that require a commit() call, - after a call to executemany(). - - fixed a bug inserting aka titles in database that checks for foreign - keys consistency. - - fixed an obscure bug splitting too huge data sets. - - MoviesCache and PersonsCache are now flushed few times. - - fixed a bug handling excessive recursion. - - improved the exceptions handling. - - -* What's the new in release 2.5 "Ninja Thunderbolt" (15 May 2006) - [general] - - support for tv series episodes; see the README.series file. - - modified the DISCLAIMER.txt file to be compliant to the debian guidelines. - - fixed a bug in the get_first_movie.py script. - - Movie and Person instances are now hashable, so that they can be used - as dictionary keys. - - modified functions analyze_title and build_title to support tv episodes. - - use isinstance for type checking. - - minor updates to the documentation. - - the imdbID for Movie and Person instances is now searched if either - one of movieID/personID and title/name is provided. - - introduced the isSame() method for both Movie and Person classes, - useful to compare object by movieID/personID and accessSystem. - - __contains__() methods are now recursive. - - two new functions in the IMDbBase class, title2imdbID() and name2imdbID() - are used to get the imdbID, given a movie title or person name. - - two new functions in the helpers module, sortedSeasons() and - sortedEpisodes(), useful to manage lists/dictionaries of tv series - episodes. - - in the helpers module, the get_byURL() function can be used to retrieve - a Movie or Person object for the given URL. - - renamed the "ratober" C module to "cutils". - - added CONTRIBUTORS.txt file. - - [http data access system] - - fixed a bug regarding currentRole for tv series. - - fixed a bug about the "merchandising links" page. - - [http and mobile data access systems] - - fixed a bug retrieving cover url for tv (mini) series. - - [mobile data access system] - - fixed a bug with tv series titles. - - retrieves the number of episodes for tv series. - - [local data access system] - - new get_episodes function in the cutils/ratober C module. - - search functions (both C and pure python) are now a lot faster. - - updated the documentation with work-arounds to make the mkdb program - works with a recent set of plain text data files. - - [sql data access system] - - uses the SQLObject ORM to support a wide range of database engines. - - added in the cutils C module the soundex() function, and a fall back - Python only version in the parser.sql package. - - -* What's the new in release 2.4 "Munich" (09 Feb 2006) - [general] - - strings are now unicode/utf8. - - unified Movie and Person classes. - - the strings used to store every kind of information about movies and - person now are modified (substituting titles and names references) - only when it's really needed. - - speed improvements in functions modifyStrings, sortMovies, - canonicalName, analyze_name, analyze_title. - - performance improvements in every data access system. - - removed the deepcopy of the data, updating Movie and Person - information. - - moved the "ratober" C module in the imdb.parser.common package, - being used by both ""http" and "sql" data access systems. - - C functions in the "ratober" module are always case insensitive. - - the setup.py script contains a work-around to make installation - go on even if the "ratober" C module can't be compiled (displaying - a warning), since it's now optional. - - minor updates to documentation, to keep it in sync with changes - in the code. - - the new helpers.py module contains functions useful to write - IMDbPY-based programs. - - new doc file README.utf8, about unicode support. - - [http data access system] - - the ParserBase class now inherits from sgmllib.SGMLParser, - instead of htmllib.HTMLParser, resulting in a little improvement - in parsing speed. - - fixed a bug in the parser for the "news" page for movies and - persons. - - removed special handlers for entity and chardefs in the HTMLMovieParser - class. - - fixed bugs related to non-ascii chars. - - fixed a bug retrieving the URL of the cover. - - fixed a nasty bug retrieving the title field. - - retrieve the 'merchandising links' page. - - support for the new "episodes cast" page for tv series. - - fixed a horrible bug retrieving guests information for tv series. - - [sql data access system] - - fixed the imdbpy2sql.py script, to handle files with spurious lines. - - searches for names and titles are now much faster, if the - imdb.parser.common.ratober C module is compiled and installed. - - imdbpy2sql.py now works also on partial data (i.e. if you've not - downloaded every single plain text file). - - imdbpy2sql.py considers also a couple of files in the contrib directory. - - searching names and titles, only the first 5 chars returned from - the SOUNDEX() SQL function are compared. - - should works if the database is set to unicode/utf-8. - - [mobile data access system] - - fixed bugs related to non-ascii chars. - - fixed a bug retrieving the URL of the cover. - - retrieve currentRole/notes also for tv guest appearances. - - [local data access system] - - it can work even if the "ratober" C module is not compiled; - obviously the pure python substitute is painfully slow (a - warning is issued). - - -* What's the new in release 2.3 "Big Fish" (03 Dec 2005) - [general] - - uniformed numerous keys for Movie and Person objects. - - 'birth name' is now always in canonical form, and 'nick names' - are always normalized; these changes also affect the sql data - access system. - - [http data access system] - - removed the 'imdb mini-biography by' key; the name of the author - is now prepended to the 'mini biography' key. - - fixed an obscure bug using more than one access system (http in - conjunction with mobile or httpThin). - - fixed a bug in amazon reviews. - - [mobile data access system] - - corrected some bugs retrieving filmography and cast list. - - [sql data access system] - - remove 'birth name' and 'nick names' from the list of 'akas'. - - in the SQL database, 'crewmembers' is now 'miscellaneous crew'. - - fixed a bug retrieving "guests" for TV Series. - - -* What's the new in release 2.2 "The Thing" (17 Oct 2005) - [general] - - now the Person class has a 'billingPos' instance variable used to - keep record of the position of the person in the list of credits (as - an example, "Laurence Fishburne" is billed in 2nd position in the - cast list for the "Matrix, The (1999)" movie. - - added two functions to the utils module, to sort respectively - movies (by year/title/imdbIndex) and persons (by billingPos/name/imdbIndex). - - every data access system support the 'adultSearch' argument and the - do_adult_search() method to exclude the adult movies from your searches. - By default, adult movies are always listed. - - renamed the scripts, appending the ".py" extension. - - added an "IMDbPY Powered" logo and a bitmap used by the Windows installer. - - now Person and Movie objects always convert name/title to the canonical - format (Title, The). - - minor changes to the functions used to convert to "canonical format" - names and titles; they should be faster and with better matches. - - 'title' is the first argument, instancing a Movie object (instead - of 'movieID'). - - 'name' is the first argument, instancing a Movie object (instead - of 'personID'). - - [http data access system] - - retrieves the 'guest appearances' page for TV series. - - fixed a bug retrieving newsgroup reviews urls. - - fixed a bug managing non-breaking spaces (they're truly a damnation!) - - fixed a bug with mini TV Series in people's biographies. - - now keywords are in format 'bullet-time' and no more 'Bullet Time'. - - [mobile data access system] - - fixed a bug with direct hits, searching for a person's name. - - fixed a bug with languages and countries. - - [local data access system] - - now cast entries are correctly sorted. - - new search system; it should return better matches in less - time (searching people's name is still somewhat slow); it's - also possibile to search for "long imdb canonical title/name". - - fixed a bug retrieving information about a movie with the same - person listed more than one time in a given role/duty (e.g., the - same director for different episodes of a TV series). Now it - works fine and it should also be a bit faster. - - 'notable tv guest appearences' in biography is now a list of Movie - objects. - - writers are sorted in the right order. - - [sql data access system] - - search results are now sorted in correct order; difflib is used to - calculate strings similarity. - - new search SQL query and comparison algorithm; it should return - much better matches. - - searches for only a surname now returns much better results. - - fixed a bug in the imdbpy2sql.py script; now movie quotes are correctly - managed. - - added another role, 'guests', for notable tv guest appearences. - - writers are sorted in the right order. - - put also the 'birth name' and the 'nick names' in the akanames table. - - -* What's the new in release 2.1 "Madagascar" (30 Aug 2005) - [general] - - introduced the "sql data access system"; now you can transfer the - whole content of the plain text data files (distributed by IMDb) - into a SQL database (MySQL, so far). - - written a tool to insert the plain text data files in a SQL database. - - fixed a bug in items() and values() methods of Movie and Person - classes. - - unified portions of code shared between "local" and "sql". - - [http data access system] - - fixed a bug in the search_movie() and search_person() methods. - - parse the "external reviews", "newsgroup reviews", "newsgroup reviews", - "misc links", "sound clips", "video clips", "amazon reviews", "news" and - "photo sites" pages for movies. - - parse the "news" page for persons. - - fixed a bug retrieving personID and movieID within namesRefs - and titlesRefs. - - [local data access system] - - fixed a bug; 'producer' data where scanned two times. - - some tags were missing for the laserdisc entries. - - [mobile data access system] - - fixed a bug retrieving cast information (sometimes introduced - with "Cast overview" and sometimes with "Credited cast"). - - fixed a bug in the search_movie() and search_person() methods. - - -* What's the new in release 2.0 "Land Of The Dead" (16 Jul 2005) - - [general] - - WARNING! Now, using http and mobile access methods, movie/person - searches will include by default adult movie titles/pornstar names. - You can still deactivate this feature by setting the adultSearch - argument to false, or calling the do_adult_search() method with - a false value. - - fixed a bug using the 'all' keyword of the 'update' method. - - [http data access system] - - added the "recommendations" page. - - the 'notes' instance variable is now correctly used to store - miscellaneous information about people in non-cast roles, replacing - the 'currentRole' variable. - - the adultSearch initialization argument is by default true. - - you can supply the proxy to use with the 'proxy' initialization - argument. - - retrieve the "plot outline" information. - - fixed a bug in the BasicMovieParser class, due to changes in the - IMDb's html. - - the "rating details" parse information about the total number - of voters, arithmetic mean, median and so on. The values are - stored as integers and floats, and no more as strings. - - dictionary keys in soundtrack are lowercase. - - fixed a bug with empty 'location' information. - - [mobile data access system] - - number of votes, rating and top 250 rank are now integers/floats. - - retrieve the "plot outline" information. - - [local data access system] - - number of votes, rating and top 250 rank are now integers/floats. - - -* What's the new in release 1.9 "Ed Wood" (02 May 2005) - [general] - - introduced the new "mobile" data access system, useful for - small systems. It should be from 2 to 20 times faster than "http" - or "httpThin". - - the "http", "httpThin" and "mobile" data access system can now - search for adult movies. See the README.adult file. - - now it should works again with python 2.0 and 2.1. - - fixed a bug affecting performances/download time. - - unified some keywords amongst differents data access systems. - - [http data access system] - - fixed some bugs; now it retrieves names akas correctly. - - -* What's the new in release 1.8 "Paths Of Glory" (24 Mar 2005) - [general] - - introduced a new data access system "httpThin", useful for - systems with limited bandwidth and CPU power, like PDA, - hand-held devices and mobile phones. - - the setup.py script can be configured to not compile/install - the local access system and the example scripts (useful for - hand-held devices); introduced setup.cfg and MANIFEST.in files. - - updated the list of articles used to manage movie titles. - - removed the all_info tuples from Movie and Person classes, - since the list of available info sets depends on the access - system. I've added two methods to the IMDbBase class, - get_movie_infoset() and get_person_infoset(). - - removed the IMDbNotAvailable exception. - - unified some code in methods get_movie(), get_person() and - update() in IMDbBase class. - - minor updates to the documentation; added a 46x46 PNG icon. - - documentation for small/mobile systems. - - [Movie class] - - renamed the m['notes'] item of Movie objects to m['episodes']. - - [Person class] - - the p.__contains__(m) method can be used to check if the p - Person has worked in the m Movie. - - [local data access system] - - gather information about "laserdisc", "literature" and "business". - - fixed a bug in ratober.c; now the search_name() function - handles search strings already in the "Surname, Name" format. - - two new methods, get_lastMovieID() and get_lastPersonID(). - - [http data access system] - - limit the number of results for the query; this will save a - lot of bandwidth. - - fixed a bug retrieving the number of episodes of tv series. - - now it retrieves movies information about "technical specifications", - "business data", "literature", "soundtrack", "dvd" and "locations". - - retrieves people information about "publicity" and "agent". - - -* What's the new in release 1.7 "Saw" (04 Feb 2005) - [general] - - Person class has two new keys; 'canonical name' and - 'long imdb canonical name', like "Gibson, Mel" and - "Gibson, Mel (I)". - - now titles and names are always internally stored in the - canonical format. - - search_movie() and search_person() methods return the - "read" movieID or personID (handling aliases). - - Movie and Person objects have a 'notes' instance attribute, - used to specify comments about the role of a person in a movie. - The Movie class can also contain a ['notes'] item, used to - store information about the runtime; e.g. (26 episodes). - - fixed minor bugs in the IMDbBase, Person and Movie classes. - - some performance improvements. - - [http data access system] - - fixed bugs retrieving the currentRole. - - try to handle unicode chars; return unicode strings when required. - - now the searches return also "popular titles" and - "popular names" from the new IMDb's search system. - - [local data access system] - - information about movie connections are retrieved. - - support for multiple biographies. - - now it works with Python 2.2 or previous versions. - - fixed a minor glitch in the initialization of the ratober C module. - - fixed a pair buffer overflows. - - fixed some (very rare) infinite loops bugs. - - it raises IMDbDataAccessError for (most of) I/O errors. - - [Movie class] - - fixed a bug getting the "long imdb canonical title". - - -* What's the new in release 1.6 "Ninja Commandments" (04 Jan 2005) - [general] - - now inside Movie and Person object, the text strings (biography, - movie plot, etc.) contain titles and names references, like - "_Movie, The (1999)_ (qv)" or "'A Person' (qv)"; these reference - are transformed at access time with a user defined function. - - introduced _get_real_movieID and _get_real_personID methods - in the IMDbBase class, to handle title/name aliases for the - local access system. - - split the _normalize_id method in _normalize_movieID - and _normalize_personID. - - fixed some bugs. - - [Movie class] - - now you can access the 'canonical title' and - 'long imdb canonical title' attributes, to get the movie title - in the format "Movie Title, The". - - [local data access system] - - title and name aliases now work correctly. - - now get_imdbMovieID and get_imdbPersonID methods should - work in almost every case. - - people's akas are handled. - - [http data access system] - - now the BasicMovieParser class can correctly gather the imdbID. - - -* What's the new in release 1.5 "The Incredibles" (23 Dec 2004) - [local database] - - support a local installation of the IMDb database! - WOW! Now you can download the plain text data files from - http://imdb.com/interfaces.html and access those - information through IMDbPY! - - [general] - - movie titles and person names are "fully normalized"; - Not "Matrix, The (1999)", but "The Matrix (1999)"; - Not "Cruise, Tom" but "Tom Cruise". - - get_mop_infoSet() methods can now return a tuple with the - dictionary data and a list of information sets they provided. - - [http data access system] - - support for the new search system (yes, another one...) - - a lot of small fixes to stay up-to-date with the html - of the IMDb web server. - - modified the personParser module so that it will no - more download both "filmoyear" and "maindetails" pages; - now only the latter is parsed. - - movie search now correctly reports the movie year and index. - - gather "locations" information about a movie. - - modified the HTMLAwardsParser class so that it doesn't list - empty entries. - - -* What's the new in release 1.4 "The Village" (10 Nov 2004) - [http data access system] - - modified the personParser.HTMLMaindetailsParser class, - because IMDb has changed the img tag for the headshot. - - now 'archive footage' is handled correctly. - - [IMDb class] - - fixed minor glitches (missing "self" parameter in a - couple of methods). - - [misc] - - now distutils installs also the example scripts in ./bin/* - - -* What's the new in release 1.3 "House of 1000 Corpses" (6 Jul 2004) - [http data access system] - - modified the BasicMovieParser and BasicPersonParser classes, - because IMDb has removed the "pageflicker" from the html pages. - - [general] - - the test suite was moved outside the tgz package. - - -* What's the new in release 1.2 "Kill Bill" (2 May 2004) - [general] - - now it retrieves almost every available information about movie - and people! - - introduced the concept of "data set", to retrieve different sets - of information about a movie/person (so that it's possibile to - fetch only the needed information). - - introduced a test suite, using the PyUnit (unittest) module. - - fixed a nasty typo; the analyze_title and build_title functions - now use the strings 'tv mini series' and 'tv series' for the 'kind' - key (previously the 'serie' word ws used). - - new design; removed the mix-in class and used a factory pattern; - imdb.IMDb is now a function, which returns an instance of a class, - subclass of imdb.IMDbBase. - - introduced the build_name(name_dict) function in the utils module, - which takes a dictionary and build a long imdb name. - - fixed bugs in the analyze_name function; now it correctly raise - an IMDbParserError exception for empty/all spaces strings. - - now the analyze_title function sets only the meaningful - information (i.e.: no 'kind' or 'year' key, if they're not set) - - [http data access system] - - removed all non-greedy regular expressions. - - removed all regular expressions in the movieParser module; now - self.rawdata is no more used to search "strange" matches. - - introduced a ParserBase class, used as base class for the parsers. - - retrieve information about the production status (pre-production, - announced, in production, etc.) - - mpaa is now a string. - - now when an IMDbDataAccessError is raised it shows also the - used proxy. - - minor changes to improve performances in the handle_data method of - the HTMLMovieParser class. - - minor changes to achieve a major performances improvement in - the BasicPersonParser class in the searchPersonParse module. - - [Movie class] - - fixed a bug in isSameTitle method, now the accessSystem is correctly - checked. - - fixed some typos. - - [Person class] - - minor changes to the isSamePerson method (now it uses the build_name - function). - - -* What's the new in release 1.1 "Gigli" (17 Apr 2004) - [general] - - added support for persons (search & retrieve information about people). - - removed the dataSets module. - - removed the MovieTitle and the SearchMovieResults classes; now information - about the title is stored directly in the Movie object and the search - methods return simple lists (of Movie or Person objects). - - removed the IMDbTitleError exception. - - added the analyze_name() function in the imdb.utils module, which - returns a dictionary with the 'name' and 'imdbIndex' keys from the - given long imdb name string. - - [http data access system] - - http search uses the new search system. - - moved the plotParser module content inside the movieParser module. - - fixed a minor bug handling AKAs for movie titles. - - [IMDb class] - - introduced the update(obj) method of the IMDb class, to update - the information of the given object (a Movie or Person instance). - - added the get_imdbURL(obj) method if the IMDb class, which returns - the URL of the main IMDb page for the given object (a Movie or Person). - - renamed the 'kind' parameter of the IMDb class to 'accessSystem'. - - [Movie class] - - now __str__() returns only the short name; the summary() method - returns a pretty-printed string for the Movie object. - - persons are no more simple strings, but Person objects (the role/duty - is stored in the currentRole variable of the object). - - isSameTitle(obj) method to compare two Movie objects even when - not all information are gathered. - - new __contains__() method, to check is a given person was in a movie. - - [misc] - - updated the documentation. - - corrected some syntax/grammar errors. - - -* What's the new in release 1.0 "Equilibrium" (01 Apr 2004) - [general] - - first public release. - - retrieve data only from the web server. - - search only for movie titles. - - diff -pruN 5.1-1/docs/conf.py 6.6-1/docs/conf.py --- 5.1-1/docs/conf.py 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/conf.py 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'IMDbPY' +copyright = '2018, Davide Alberani, H. Turgut Uyar' +author = 'Davide Alberani, H. Turgut Uyar' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '6.6' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +modindex_common_prefix = ['imdb.'] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'IMDbPYdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'IMDbPY.tex', 'IMDbPY Documentation', + 'Davide Alberani, H. Turgut Uyar', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'imdbpy', 'IMDbPY Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'IMDbPY', 'IMDbPY Documentation', + author, 'IMDbPY', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- diff -pruN 5.1-1/docs/contributors/credits.rst 6.6-1/docs/contributors/credits.rst --- 5.1-1/docs/contributors/credits.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/contributors/credits.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,360 @@ +Credits +------- + +First of all, I want to thank all the package maintainers, and especially +Ana Guerrero. Another big thanks to the developers who used IMDbPY +for their projects and research; they can be found here: +https://imdbpy.sourceforge.io/ecosystem.html + +Other very special thanks go to some people who followed the development +of IMDbPY very closely, providing hints and insights: Ori Cohen, James Rubino, +Tero Saarni, and Jesper Noer (for a lot of help, and also for the wonderful +https://bitbucket.org/); and let's not forget all the translators +on https://www.transifex.com/davide_alberani/imdbpy/. + +Below is a list of people who contributed with bug reports, small patches, +and hints (kept in reverse order since IMDbPY 4.5): + +* Ali Momen Sani for a report about mini biography + +* David Runge for package tests and hints + +* antonioforte for a bug report about XML output + +* Jakub Synowiec for multiple bug reports and patches + +* Piotr Staszewski for multiple bug reports and patches + +* tiagoaquinofl and rednibia for extensive debugging on SSL certificate issues + +* Tim King for a report about birth and death dates. + +* Lars Gustäbel for a report about series seasons. + +* Filip BaÄić for a report about full-size headshot + +* Matthew Clapp for a report about pip installation + +* Jannik S for a report on tech parser + +* Brad Pirtle, Adrien C. and Markus-at-GitHub for improvements to full-size covers + +* Tim Belcher for a report about forgotten debug code. + +* Paul Jensen for many bug reports about tv series. + +* Andrew D Bate for documentation on how to reintroduce foreign keys. + +* yiqingzhu007 for a bug report about synopsis. + +* David Runge for managing the Arch Linux package. + +* enriqueav for fixes after the IMDb redesign. + +* Piotr Staszewski for a fix for external sites parser. + +* Mike Christopher for the user reviews parser. + +* apelord for a parser for full credits. + +* Mike Christopher for a patch for synopsis parser. + +* Damon Brodie for a bug report about technical parser. + +* Sam Petulla for a bug report about searching for keywords. + +* zoomorph for an improvement for parsing your own votes. + +* Fabrice Laporte for a bug report on setup.py. + +* Wael Sinno for a patch for parsing movie-links. + +* Tool Man, for a fix on sound track parsing. + +* Rafael Lopez for a series of fixes on top/bottom lists. + +* Derek Duoba for a bug report about XML output. + +* Cody Hatfield for a parser for the Persons's resume page. + +* Mystfit for a fix handling company names. + +* Troy Deck for a path for MySQL. + +* miles82 for a patch on metascore parsing. + +* Albert Claret for the parser of the critic reviews page. + +* Shobhit Singhal for fixes in parsing biographies and plots. + +* Dan Poirier for documentation improvements. + +* Frank Braam for a fix for MSSQL. + +* Darshana Umakanth for a bug report the search functions. + +* Osman Boyaci for a bug report on movie quotes. + +* Mikko Matilainen for a patch on encodings. + +* Roy Stead for the download_applydiffs.py script. + +* Matt Keenan for a report about i18n in search results. + +* belgabor for a patch in the goofs parser. + +* Ian Havelock for a bug report on charset identification. + +* Mikael Puhakka for a bug report about foreign language results in a search. + +* Wu Mao for a bug report on the GAE environment. + +* legrostdg for a bug report on the new search pages. + +* Haukur Páll Hallvarðsson for a patch on query parameters. + +* Arthur de Peretti-Schlomoff for a list of French articles and + fixes to Spanish articles. + +* John Lambert, Rick Summerhill and Maciej for reports and fixes + for the search query. + +* Kaspars "Darklow" Sprogis for an impressive amount of tests and reports about + bugs parsing the plain text data files and many new ideas. + +* Damien Stewart for many bug reports about the Windows environment. + +* Vincenzo Ampolo for a bug report about the new imdbIDs save/restore queries. + +* Tomáš Hnyk for the idea of an option to reraise caught exceptions. + +* Emmanuel Tabard for ideas, code and testing on restoring imdbIDs. + +* Fabian Roth for a bug report about the new style of episodes list. + +* Y. Josuin for a bug report on missing info in crazy credits file. + +* Arfrever Frehtes Taifersar Arahesis for a patch for locales. + +* Gustaf Nilsson for bug reports about BeautifulSoup. + +* Jernej Kos for patches to handle "in production" information + and birth/death years. + +* Saravanan Thirumuruganathan for a bug report about genres in mobile. + +* Paul Koan, for a bug report about DVD pages and movie references. + +* Greg Walters for a report about a bug with queries with too + many results. + +* Olav Kolbu for tests and report about how the IMDb.com servers + reply to queries made with and without cookies. + +* Jef "ofthelit", for a patch for the reduce.sh script bug + reports for Windows. + +* Reiner Herrmann for benchmarks using SSD hard drives. + +* Thomas Stewart for some tests and reports about a bug + with charset in the plain text data files. + +* Ju-Hee Bae for an important series of bug reports about + the problems derived by the last IMDb's redesign. + +* Luis Liras and Petite Abeille for a report and a bugfix about + imdbpy2sql.py used with SQLite and SQLObject. + +* Kevin S. Anthony for a bug report about episodes list. + +* Bhupinder Singh for a bug report about exception handling in Python 2.4. + +* Ronald Hatcher for a bug report on the GAE environment. + +* Ramusus for a lot of precious bug reports. + +* Laurent Vergne for a hint about InnoDB, MyISAM and foreign keys. + +* Israel Fruch for patches to support the new set of parsers. + +* Inf3cted MonkeY, for a bug report about 'vote details'. + +* Alexmipego, for suggesting to add a md5sum to titles and names. + +* belgabortm for a bug report about movies with multiple 'countries'. + +* David Kaufman for an idea to make the 'update' method more robust. + +* Dustin Wyatt for a bug with SQLite of Python 2.6. + +* Julian Scheid for bug reports about garbage in the ptdf. + +* Adeodato Simó for a bug report about the new imdb.com layout. + +* Josh Harding for a bug report about the new imdb.com layout. + +* Xavier Naidoo for a bug report about top250 and BeautifulSoup. + +* Basil Shubin for hints about a new helper function. + +* Mark Jeffery, for some help debugging a lxml bug. + +* Hieu Nguyen for a bug report about fetching real imdbIDs. + +* Rdian06 for a patch for movies without plot authors. + +* Tero Saarni, for the series 60 GUI and a lot of testing and + debugging. + +* Ana Guerrero, for maintaining the official debian package. + +* H. Turgut Uyar for a number of bug reports and a lot of work on + the test-suite. + +* Ori Cohen for some code and various hints. + +* Jesper Nøhr for a lot of testing, especially on 'sql'. + +* James Rubino for many bug reports. + +* Cesare Lasorella for a bug report about newer versions of SQLObject. + +* Andre LeBlanc for a bug report about airing date of tv series episodes. + +* aow for a note about some misleading descriptions. + +* Sébastien Ragons for tests and reports. + +* Sridhar Ratnakumar for info about PKG-INF. + +* neonrush for a bug parsing Malcolm McDowell filmography! + +* Alen Ribic for some bug reports and hints. + +* Joachim Selke for some bug reports with SQLAlchemy and DB2 and a lot + of testing and debugging of the ibm_db driver (plus a lot of hints + about how to improve the imdbpy2sql.py script). + +* Karl Newman for bug reports about the installer of version 4.5. + +* Saruke Kun and Treas0n for bug reports about 'Forbidden' errors + from the imdb.com server. + +* Chris Thompson for some bug reports about summary() methods. + +* Mike Castle for performace tests with SQLite and numerous hints. + +* Indy (indyx) for a bug about series cast parsing using BeautifulSoup. + +* Yoav Aviram for a bug report about tv mini-series. + +* Arjan Gijsberts for a bug report and patch for a problem with + movies listed in the Bottom 100. + +* Helio MC Pereira for a bug report about unicode. + +* Michael Charclo for some bug reports performing 'http' queries. + +* Amit Belani for bug reports about plot outline and other changes. + +* Matt Warnock for some tests with MySQL. + +* Mark Armendariz for a bug report about too long field in MySQL db + and some tests/analyses. + +* Alexy Khrabrov, for a report about a subtle bug in imdbpy2sql.py. + +* Clark Bassett for bug reports and fixes about the imdbpy2sql.py + script and the cutils.c C module. + +* mumas for reporting a bug in summary methods. + +* Ken R. Garland for a bug report about 'cover url' and a lot of + other hints. + +* Steven Ovits for hints and tests with Microsoft SQL Server, SQLExpress + and preliminary work on supporting diff files. + +* Fredrik Arnell for tests and bug reports about the imdbpy2sql.py script. + +* Arnab for a bug report in the imdbpy2sql.py script. + +* Elefterios Stamatogiannakis for the hint about transactions and SQLite, + to obtain an impressive improvement in performances. + +* Jon Sabo for a bug report about unicode and the imdbpy2sql.py script + and some feedback. + +* Andrew Pendleton for a report about a very hideous bug in + the imdbpy2sql.py (garbage in the plain text data files + programming + errors + utf8 strings + postgres). + +* Ataru Moroboshi ;-) for a bug report about role/duty and notes. + +* Ivan Kedrin for a bug report about the analyze_title function. + +* Hadley Rich for reporting bugs and providing patches for troubles + parsing tv series' episodes and searching for tv series' titles. + +* Jamie R. Rytlewski for a suggestion about saving imbIDs in 'sql'. + +* Vincent Crevot, for a bug report about unicode support. + +* Jay Klein for a bug report and testing to fix a nasty bug in the + imdbpy2sql.py script (splitting too large data sets). + +* Ivan Garcia for an important bug report about the use of IMDbPY + within wxPython programs. + +* Kessia Pinheiro for a bug report about tv series list of episodes. + +* Michael G. Noll for a bug report and a patch to fix a bug + retrieving 'plot keywords'. + +* Alain Michel, for a bug report about search_*.py and get_*.py scripts. + +* Martin Arpon and Andreas Schoenle for bug reports (and patches) + about "runtime", "aka titles" and "production notes" information + not being parsed. + +* none none (dclist at gmail.com) for a useful hint and code to + retrieve a movie/person object, given an URL. + +* Sebastian Pölsterl, for a bug report about the cover url for + tv (mini) series, and another one about search_* methods. + +* Martin Kirst for many hints and the work on the imdbpyweb program. + +* Julian Mayer, for a bug report and a patch about non-ascii chars. + +* Wim Schut and "eccentric", for bug reports and a patches about + movies' cover url. + +* Alfio Ferrara, for a bug report about the get_first_movie.py script. + +* Magnus Lie Hetland for an hint about the searches in sql package. + +* Thomas Jadjewski for a bug report about the imdbpy2sql.py script. + +* Trevor MacPhail, for a bug report about search_* methods and + the ParserBase.parse method. + +* Guillaume Wisniewski, for a bug report. + +* Kent Johnson, for a bug report. + +* Andras Bali, for the hint about the "plot outline" information. + +* Nick S. Novikov, who provided the Windows installer until I've + managed to set up a Windows development environment. + +* Simone Bacciglieri, who downloaded the plain text data files for me. + +* Carmine Noviello, for some design hints. + +* "Basilius" for a bug report. + +* Davide for a bug report. + + +.. _Contributors: CONTRIBUTORS.html diff -pruN 5.1-1/docs/contributors/index.rst 6.6-1/docs/contributors/index.rst --- 5.1-1/docs/contributors/index.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/contributors/index.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,93 @@ +Contributors +============ + +Authors +------- + +People who contributed a substantial amount of work and share the copyright +over some portions of the code: + +Davide Alberani erlug.linux.it> + + Main author and project leader. + + +\H. Turgut Uyar tekir.org> + + The whole "http" data access system (using a DOM and XPath-based + approach) is based on his work. The imdbpykit interface was mostly written + by him and he holds the copyright over the whole code (with some portions + shared with others). He provided the tox testsuite. + + +Giuseppe "Cowo" Corbelli lugbs.linux.it> + + Provided a lot of code and hints to integrate IMDbPY with SQLObject, + working on the imdbpy2sql.py script and the dbschema.py module. + + +Beside Turgut, Giuseppe and me, the following people are listed as developers +for the IMDbPY project on sourceforge and may share copyright on some (minor) +portions of the code: + + +Alberto Malagoli + + Developed the new web site, and detains the copyright of it, + and provided helper functions and other code. + + +Martin Kirst s1998.tu-chemnitz.de> + + Has done an important refactoring of the imdbpyweb program + and shares with me the copyright on the whole program. + + +Jesper Nøhr noehr.org> + + Provided extensive testing and some patches for the "http" + data access system. + + +Joachim Selke tu-bs.de> + + Many tests on IBM DB2 and work on the CSV support. + + +Timo Schulz users.sourceforge.net> + + Did a lot of work "sql", DB2 and CSV support and extensive analysis + aimed at diff files support. + + +Roy Stead gmail.com> + + Provided the download_applydiffs.py script. + + +Translators +----------- + +Additional translations were provided by: + +- strel (Spanish) +- Stéphane Aulery (French) +- RainDropR (Arabic) +- Atanas Kovachki (Bulgarian) +- lukophron (French) +- Raphael (German) + + +.. include:: credits.rst + + +Donations +--------- + +We'd like to thank the following people for their donations: + +- Paulina Wadecka +- Oleg Peil +- Diego Sarmentero +- Fabian Winter +- Lacroix Scott diff -pruN 5.1-1/docs/CONTRIBUTORS.txt 6.6-1/docs/CONTRIBUTORS.txt --- 5.1-1/docs/CONTRIBUTORS.txt 2015-09-07 19:17:54.000000000 +0000 +++ 6.6-1/docs/CONTRIBUTORS.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,78 +0,0 @@ - OTHER AUTHORS - ============= - -People who contributed with a substantial amount of work and that -share the copyright over some portions of the code: - -NAME: H. Turgut Uyar -EMAIL: tekir.org> -CONTRIBUTION: the whole new "http" data access system (using a DOM and -XPath-based approach) is based on his work. The imdbpykit interface -was mostly written by him and he holds the copyright over the whole -code (with some portions shared with others). - - -NAME: Giuseppe "Cowo" Corbelli -EMAIL: lugbs.linux.it> -CONTRIBUTION: provided a lot of code and hints to integrate IMDbPY -with SQLObject, working on the imdbpy2sql.py script and the dbschema.py -module. - - -Actually, besides Turgut, Giuseppe and me, these other people are -listed as developers for the IMDbPY project on sourceforge and may -share copyright on some (minor) portions of the code: - -NAME: Alberto Malagoli -CONTRIBUTION: developed the new web site, and detains the copyright of it, -and provided helper functions and other code. - - -NAME: Martin Kirst -EMAIL: s1998.tu-chemnitz.de> -CONTRIBUTION: has done an important refactoring of the imdbpyweb -program and shares with me the copyright on the whole program. - - -NAME: Jesper Nøhr -EMAIL: noehr.org> -CONTRIBUTION: provided extensive testing and some patches for -the 'http' data access system. - - -NAME: Joachim Selke -EMAIL: tu-bs.de> -CONTRIBUTION: many tests on IBM DB2 and work on the CSV support. - - -NAME: Timo Schulz -EMAIL: users.sourceforge.net> -CONTRIBUTION: did a lot of work 'sql', DB2 and CSV support and -extensive analysis aimed at diff files support. - -NAME: Roy Stead -EMAIL: gmail.com> -CONTRIBUTION: provided the download_applydiffs.py script. - - - TRANSLATORS - =========== - -Additional translations were provided by: - - strel (Spanish) - - Stéphane Aulery (French) - - RainDropR (Arabic) - - Atanas Kovachki (Bulgarian) - - lukophron (French) - - Raphael (German) - - - DONATIONS - ========= - -We'd like to thank these persons for their donations: - - Oleg Peil - - Diego Sarmentero - - Fabian Winter - - Lacroix Scott - diff -pruN 5.1-1/docs/CREDITS.txt 6.6-1/docs/CREDITS.txt --- 5.1-1/docs/CREDITS.txt 2016-10-23 16:06:24.000000000 +0000 +++ 6.6-1/docs/CREDITS.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,321 +0,0 @@ - - CREDITS - ======= - -See also CONTRIBUTORS.txt for a list of the most important developers -who share the copyright on some portions of the code. - -First of all, I want to thank all the maintainers of the -packages, listed on http://imdbpy.sf.net/?page=download#otherpkg, -and especially Ana Guerrero. -Another big thank to the developers who used IMDbPY for their -projects and researches; they can be found here: - http://imdbpy.sf.net/?page=programs - -Other very special thanks go to some people who followed very -closely the development of IMDbPY, providing hints and insights: -Ori Cohen, James Rubino, Tero Saarni and Jesper Noer (for a lot -of help, and also for the wonderful http://bitbucket.org), and -let's not forget all the translators on https://www.transifex.com/davide_alberani/imdbpy/ - - -Below, a list of persons who contributed with bug reports, small -patches and hints (kept in a reverse order since IMDbPY 4.5): - -* Damon Brodie for a bug report about technical parser. - -* Sam Petulla for a bug report about searching for keywords - -* zoomorph for an improvement for parsing your own votes. - -* Fabrice Laporte for a bug report on setup.py - -* Wael Sinno for a patch for parsing movie-links. - -* Tool Man, for a fix on sound track parsing. - -* Rafael Lopez for a series of fixes on top/bottom lists. - -* Derek Duoba for a bug report about XML output. - -* Cody Hatfield for a parser for the Persons's resume page - -* Mystfit for a fix handling company names. - -* Troy Deck for a path for MySQL. - -* miles82 for a patch on metascore parsing. - -* Albert Claret for the parser of the critic reviews page. - -* Shobhit Singhal for fixes in parsing biographies and plots. - -* Dan Poirier for documentation improvements. - -* Frank Braam for a fix for MSSQL. - -* Darshana Umakanth for a bug report the search functions. - -* Osman Boyaci for a bug report on movie quotes. - -* Mikko Matilainen for a patch on encodings. - -* Roy Stead for the download_applydiffs.py script. - -* Matt Keenan for a report about i18n in search results. - -* belgabor for a patch in the goofs parser. - -* Ian Havelock for a bug report on charset identification. - -* Mikael Puhakka for a bug report about foreign languages results doing a search. - -* Wu Mao for a bug report on the GAE environment. - -* legrostdg for a bug report on the new search pages. - -* Haukur Páll Hallvarðsson for a patch on query parameters. - -* Arthur de Peretti-Schlomoff for a list of French articles and - fixes to Spanish articles. - -* John Lambert, Rick Summerhill and Maciej for reports and fixes - for the search query. - -* Kaspars "Darklow" Sprogis for an impressive amount of tests and reports about - bugs parsing the plain text data files and many new ideas. - -* Damien Stewart for many bug reports about the Windows environment. - -* Vincenzo Ampolo for a bug report about the new imdbIDs save/restore queries. - -* Tomáš Hnyk for the idea of an option to reraise caught exceptions. - -* Emmanuel Tabard for ideas, code and testing on restoring imdbIDs. - -* Fabian Roth for a bug report about the new style of episodes list. - -* Y. Josuin for a bug report on missing info in crazy credits file. - -* Arfrever Frehtes Taifersar Arahesis for a patch for locales. - -* Gustaf Nilsson for bug reports about BeautifulSoup. - -* Jernej Kos for patches to handle "in production" information - and birth/death years. - -* Saravanan Thirumuruganathan for a bug report about genres in mobile. - -* Paul Koan, for a bug report about DVD pages and movie references. - -* Greg Walters for a report about a bug with queries with too - many results. - -* Olav Kolbu for tests and report about how the IMDb.com servers - reply to queries made with and without cookies. - -* Jef "ofthelit", for a patch for the reduce.sh script bug - reports for Windows. - -* Reiner Herrmann for benchmarks using SSD hard drives. - -* Thomas Stewart for some tests and reports about a bug - with charset in the plain text data files. - -* Ju-Hee Bae for an important series of bug reports about - the problems derived by the last IMDb's redesign. - -* Luis Liras and Petite Abeille for a report and a bugfix about - imdbpy2sql.py used with SQLite and SQLObject. - -* Kevin S. Anthony for a bug report about episodes list. - -* Bhupinder Singh for a bug report about exception handling in Python 2.4. - -* Ronald Hatcher for a bug report on the GAE environment. - -* Ramusus for a lot of precious bug reports. - -* Laurent Vergne for a hint about InnoDB, MyISAM and foreign keys. - -* Israel Fruch for patches to support the new set of parsers. - -* Inf3cted MonkeY, for a bug report about 'vote details'. - -* Alexmipego, for suggesting to add a md5sum to titles and names. - -* belgabortm for a bug report about movies with multiple 'countries'. - -* David Kaufman for an idea to make the 'update' method more robust. - -* Dustin Wyatt for a bug with SQLite of Python 2.6. - -* Julian Scheid for bug reports about garbage in the ptdf. - -* Adeodato Simó for a bug report about the new imdb.com layout. - -* Josh Harding for a bug report about the new imdb.com layout. - -* Xavier Naidoo for a bug report about top250 and BeautifulSoup. - -* Basil Shubin for hints about a new helper function. - -* Mark Jeffery, for some help debugging a lxml bug. - -* Hieu Nguyen for a bug report about fetching real imdbIDs. - -* Rdian06 for a patch for movies without plot authors. - -* Tero Saarni, for the series 60 GUI and a lot of testing and - debugging. - -* Ana Guerrero, for maintaining the official debian package. - -* H. Turgut Uyar for a number of bug reports and a lot of work on - the test-suite. - -* Ori Cohen for some code and various hints. - -* Jesper Nøhr for a lot of testing, especially on 'sql'. - -* James Rubino for many bug reports. - -* Cesare Lasorella for a bug report about newer versions of SQLObject. - -* Andre LeBlanc for a bug report about airing date of tv series episodes. - -* aow for a note about some misleading descriptions. - -* Sébastien Ragons for tests and reports. - -* Sridhar Ratnakumar for info about PKG-INF. - -* neonrush for a bug parsing Malcolm McDowell filmography! - -* Alen Ribic for some bug reports and hints. - -* Joachim Selke for some bug reports with SQLAlchemy and DB2 and a lot - of testing and debugging of the ibm_db driver (plus a lot of hints - about how to improve the imdbpy2sql.py script). - -* Karl Newman for bug reports about the installer of version 4.5. - -* Saruke Kun and Treas0n for bug reports about 'Forbidden' errors - from the imdb.com server. - -* Chris Thompson for some bug reports about summary() methods. - -* Mike Castle for performace tests with SQLite and numerous hints. - -* Indy (indyx) for a bug about series cast parsing using BeautifulSoup. - -* Yoav Aviram for a bug report about tv mini-series. - -* Arjan Gijsberts for a bug report and patch for a problem with - movies listed in the Bottom 100. - -* Helio MC Pereira for a bug report about unicode. - -* Michael Charclo for some bug reports performing 'http' queries. - -* Amit Belani for bug reports about plot outline and other changes. - -* Matt Warnock for some tests with MySQL. - -* Mark Armendariz for a bug report about too long field in MySQL db - and some tests/analyses. - -* Alexy Khrabrov, for a report about a subtle bug in imdbpy2sql.py. - -* Clark Bassett for bug reports and fixes about the imdbpy2sql.py - script and the cutils.c C module. - -* mumas for reporting a bug in summary methods. - -* Ken R. Garland for a bug report about 'cover url' and a lot of - other hints. - -* Steven Ovits for hints and tests with Microsoft SQL Server, SQLExpress - and preliminary work on supporting diff files. - -* Fredrik Arnell for tests and bug reports about the imdbpy2sql.py script. - -* Arnab for a bug report in the imdbpy2sql.py script. - -* Elefterios Stamatogiannakis for the hint about transactions and SQLite, - to obtain an impressive improvement in performances. - -* Jon Sabo for a bug report about unicode and the imdbpy2sql.py script - and some feedback. - -* Andrew Pendleton for a report about a very hideous bug in - the imdbpy2sql.py (garbage in the plain text data files + programming - errors + utf8 strings + postgres). - -* Ataru Moroboshi ;-) for a bug report about role/duty and notes. - -* Ivan Kedrin for a bug report about the analyze_title function. - -* Hadley Rich for reporting bugs and providing patches for troubles - parsing tv series' episodes and searching for tv series' titles. - -* Jamie R. Rytlewski for a suggestion about saving imbIDs in 'sql'. - -* Vincent Crevot, for a bug report about unicode support. - -* Jay Klein for a bug report and testing to fix a nasty bug in the - imdbpy2sql.py script (splitting too large data sets). - -* Ivan Garcia for an important bug report about the use of IMDbPY - within wxPython programs. - -* Kessia Pinheiro for a bug report about tv series list of episodes. - -* Michael G. Noll for a bug report and a patch to fix a bug - retrieving 'plot keywords'. - -* Alain Michel, for a bug report about search_*.py and get_*.py scripts. - -* Martin Arpon and Andreas Schoenle for bug reports (and patches) - about "runtime", "aka titles" and "production notes" information - not being parsed. - -* none none (dclist at gmail.com) for a useful hint and code to - retrieve a movie/person object, given an URL. - -* Sebastian Pölsterl, for a bug report about the cover url for - tv (mini) series, and another one about search_* methods. - -* Martin Kirst for many hints and the work on the imdbpyweb program. - -* Julian Mayer, for a bug report and a patch about non-ascii chars. - -* Wim Schut and "eccentric", for bug reports and a patches about - movies' cover url. - -* Alfio Ferrara, for a bug report about the get_first_movie.py script. - -* Magnus Lie Hetland for an hint about the searches in sql package. - -* Thomas Jadjewski for a bug report about the imdbpy2sql.py script. - -* Trevor MacPhail, for a bug report about search_* methods and - the ParserBase.parse method. - -* Guillaume Wisniewski, for a bug report. - -* Kent Johnson, for a bug report. - -* Andras Bali, for the hint about the "plot outline" information. - -* Nick S. Novikov, who provided the Windows installer until I've - managed to set up a Windows development environment. - -* Simone Bacciglieri, who downloaded the plain text data files for me. - -* Carmine Noviello, for some design hints. - -* "Basilius" for a bug report. - -* Davide for a bug report. - diff -pruN 5.1-1/docs/devel/extend.rst 6.6-1/docs/devel/extend.rst --- 5.1-1/docs/devel/extend.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/devel/extend.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,172 @@ +How to extend +------------- + +To introduce a new data access system, you have to write a new package +inside the "parser" package; this new package must provide a subclass +of the imdb.IMDb class which must define at least the following methods: + +``_search_movie(title)`` + To search for a given title; must return a list of (movieID, {movieData}) + tuples. + +``_search_episode(title)`` + To search for a given episode title; must return a list of + (movieID, {movieData}) tuples. + +``_search_person(name)`` + To search for a given name; must return a list of (movieID, {personData}) + tuples. + +``_search_character(name)`` + To search for a given character's name; must return a list of + (characterID, {characterData}) tuples. + +``_search_company(name)`` + To search for a given company's name; must return a list of + (companyID, {companyData}) tuples. + +``get_movie_*(movieID)`` + A set of methods, one for every set of information defined for a Movie + object; should return a dictionary with the relative information. + + This dictionary can contain some optional keys: + + - 'data': must be a dictionary with the movie info + - 'titlesRefs': a dictionary of 'movie title': movieObj pairs + - 'namesRefs': a dictionary of 'person name': personObj pairs + +``get_person_*(personID)`` + A set of methods, one for every set of information defined for a Person + object; should return a dictionary with the relative information. + +``get_character_*(characterID)`` + A set of methods, one for every set of information defined for a Character + object; should return a dictionary with the relative information. + +``get_company_*(companyID)`` + A set of methods, one for every set of information defined for a Company + object; should return a dictionary with the relative information. + +``_get_top_bottom_movies(kind)`` + Kind can be one of 'top' and 'bottom'; returns the related list of movies. + +``_get_keyword(keyword)`` + Return a list of Movie objects with the given keyword. + +``_search_keyword(key)`` + Return a list of keywords similar to the given key. + +``get_imdbMovieID(movieID)`` + Convert the given movieID to a string representing the imdbID, as used + by the IMDb web server (e.g.: '0094226' for Brian De Palma's + "The Untouchables"). + +``get_imdbPersonID(personID)`` + Convert the given personID to a string representing the imdbID, as used + by the IMDb web server (e.g.: '0000154' for "Mel Gibson"). + +``get_imdbCharacterID(characterID)`` + Convert the given characterID to a string representing the imdbID, as used + by the IMDb web server (e.g.: '0000001' for "Jesse James"). + +``get_imdbCompanyID(companyID)`` + Convert the given companyID to a string representing the imdbID, as used + by the IMDb web server (e.g.: '0071509' for "Columbia Pictures [us]"). + +``_normalize_movieID(movieID)`` + Convert the provided movieID in a format suitable for internal use + (e.g.: convert a string to a long int). + + NOTE: As a rule of thumb you *always* need to provide a way to convert + a "string representation of the movieID" into the internally used format, + and the internally used format should *always* be converted to a string, + in a way or another. Rationale: A movieID can be passed from the command + line, or from a web browser. + +``_normalize_personID(personID)`` + idem + +``_normalize_characterID(characterID)`` + idem + +``_normalize_companyID(companyID)`` + idem + +``_get_real_movieID(movieID)`` + Return the true movieID; useful to handle title aliases. + +``_get_real_personID(personID)`` + idem + +``_get_real_characterID(characterID)`` + idem + +``_get_real_companyID(companyID)`` + idem + +The class should raise the appropriate exceptions, when needed: + +- ``IMDbDataAccessError`` must be raised when you cannot access the resource + you need to retrieve movie info or you're unable to do a query (this is + *not* the case when a query returns zero matches: in this situation an + empty list must be returned). + +- ``IMDbParserError`` should be raised when an error occurred parsing + some data. + +Now you've to modify the ``imdb.IMDb`` function so that, when the right +data access system is selected with the "accessSystem" parameter, an instance +of your newly created class is returned. + +For example, if you want to call your new data access system "mysql" +(meaning that the data are stored in a mysql database), you have to add +to the imdb.IMDb function something like: + +.. code-block:: python + + if accessSystem == 'mysql': + from parser.mysql import IMDbMysqlAccessSystem + return IMDbMysqlAccessSystem(*arguments, **keywords) + +where "parser.mysql" is the package you've created to access the local +installation, and "IMDbMysqlAccessSystem" is the subclass of imdb.IMDbBase. + +Then it's possible to use the new data access system like: + +.. code-block:: python + + from imdb import IMDb + i = IMDb(accessSystem='mysql') + results = i.search_movie('the matrix') + print(results) + +.. note:: + + This is a somewhat misleading example: we already have a data access system + for SQL database (it's called 'sql' and it supports MySQL, amongst others). + Maybe I'll find a better example... + +A specific data access system implementation can define its own methods. +As an example, the IMDbHTTPAccessSystem that is in the parser.http package +defines the method ``set_proxy()`` to manage the use a web proxy; you can use +it this way: + +.. code-block:: python + + from imdb import IMDb + i = IMDb(accessSystem='http') # the 'accessSystem' argument is not + # really needed, since "http" is the default. + i.set_proxy('http://localhost:8080/') + +A list of special methods provided by the imdb.IMDbBase subclass, along with +their description, is always available calling the ``get_special_methods()`` +of the IMDb class: + +.. code-block:: python + + i = IMDb(accessSystem='http') + print(i.get_special_methods()) + +will print a dictionary with the format:: + + {'method_name': 'method_description', ...} diff -pruN 5.1-1/docs/devel/index.rst 6.6-1/docs/devel/index.rst --- 5.1-1/docs/devel/index.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/devel/index.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,56 @@ +Development +=========== + +If you intend to do development on the IMDbPY package, it's recommended +that you create a virtual environment for it. For example:: + + python -m venv ~/.virtualenvs/imdbpy + . ~/.virtualenvs/imdbpy/bin/activate + +In the virtual environment, install IMDbPY in editable mode and include +the extra packages. In the top level directory of the project (where +the :file:`setup.py` file resides), run:: + + pip install -e .[dev,doc,test] + + +.. packages + linguistics + Defines some functions and data useful to smartly guess the language + of a movie title (internally used). + parser (package) + A package containing a package for every data access system implemented. + http (package) + Contains the IMDbHTTPAccessSystem class which is a subclass + of the imdb.IMDbBase class; it provides the methods used to retrieve and + manage data from the web server (using, in turn, the other modules in + the package). It defines methods to get a movie and to search for a title. + The parser.sql package manages the access to the data in the SQL database, + created with the imdbpy2sql.py script; see the README.sqldb file. + The dbschema module contains tables definitions and some useful functions. + The helpers module contains functions and other goodies not directly used + by the IMDbPY package, but that can be useful to develop IMDbPY-based programs. + + +I wanted to stay independent from the source of the data for a given +movie/person/character/company, so the :func:`imdb.IMDb` function returns +an instance of a class that provides specific methods to access a given +data source (web server, SQL database, etc.). + +Unfortunately this means that the ``movieID`` +in the :class:`Movie ` class, the ``personID`` +in the :class:`Person ` class, and the ``characterID`` +in the :class:`Character ` class depend on +the data access system being used. So, when a movie, person, or character +object is instantiated, the ``accessSystem`` instance variable is set +to a string used to identify the used data access system. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + extend + test + translate + release diff -pruN 5.1-1/docs/devel/release.rst 6.6-1/docs/devel/release.rst --- 5.1-1/docs/devel/release.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/devel/release.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,105 @@ +How to make a release +===================== + +**During development** + +*setup.cfg* + + The ``egg_info`` section must include the lines below:: + + [egg_info] + tag_build = dev + tag_date = true + +*setup.py* + + The ``version`` variable must be set to the **next** version. + +*imdb/__init__.py* + + When a major fix or feature is committed, the ``VERSION`` and + ``__version__`` variables must be updated to something in the form + *{next.version}devISO8601DATE* (not mandatory, but...) + +*docs/Changelog.rst* + + When a major fix or feature is committed, the changelog must be updated. + + +**When a new release is planned** + +*setup.cfg* + + In the ``egg_info`` section, the lines mentioned above must be + commented out. + +*setup.py* + + Not touched. + +*imdb/__init__.py* + + The *devISO8601DATE* part must be removed from the version variables. + +*docs/Changelog.rst* + + The date of the release has to be added. + + +**How to release** + +- Commit the above changes. + +- Add an annotated tag like *major.minor*; e.g.: ``git tag -a 6.3`` + (the commit message is not important). + +- ``python3 setup.py sdist`` + +- ``python3 setup.py bdist_wheel`` + +- ``git push`` + +- ``git push --tags`` + +- Don't forget to push both sources and tags to both the GitHub and Bitbucket + repositories (they are kept in sync). + +- Upload to pypi: ``twine upload dist/IMDbPY-*`` (you probably need a recent + version of twine and the appropriate ~/.pypi file) + +- The new tar.gz must also be uploaded + to https://sourceforge.net/projects/imdbpy/ (along with a new "news"). + + +**communication** + +- access the web site with: `sftp ${your-sourceforge-username}@frs.sourceforge.net` and move to the *imdbpy_web/htdocs/* + +- download *index.html* and add an *article* section, removing the one or more of the old ones + +- upload the page + +- add a news on https://sourceforge.net/p/imdbpy/news/new + +- send an email to imdbpy-devel@lists.sourceforge.net and imdbpy-help@lists.sourceforge.net + + +**After the release** + +*setup.cfg* + + Uncomment the two lines again. + +*setup.py* + + Bump the ``version`` variable. + +*imdb/__init__.py* + + Bump the ``VERSION`` and ``__version__`` variables. + +*docs/Changelog.rst* + + Add a new section for the next release, on top. + +After that, you can commit the above changes with a message like "version bump" diff -pruN 5.1-1/docs/devel/test.rst 6.6-1/docs/devel/test.rst --- 5.1-1/docs/devel/test.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/devel/test.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,80 @@ +.. _testing: + +How to test +=========== + +IMDbPY has a test suite based on `pytest`_. The simplest way to run the tests +is to run the following command in the top level directory of the project:: + + pytest + +You can execute a specific test module:: + + pytest tests/test_http_movie_combined.py + +Or execute test functions that match a given keyword:: + + pytest -k cover + + +make +---- + +A :file:`Makefile` is provided for easier invocation of jobs. +The following targets are defined (among others, run "make" to see +the full list): + +test + Run tests quickly with the default Python. + +lint + Check style with flake8. + +docs + Generate Sphinx HTML documentation, including API docs. + +coverage + Check code coverage quickly with the default Python. + +clean + Clean everything. + + +tox +--- + +Multiple test environments can be tested using tox:: + + tox + +This will test all the environments listed in the :file:`tox.ini` file. +If you want to run all tests for a specific environment, for example python 3.4, +supply it as an argument to tox:: + + tox -e py34 + +You can supply commands that will be executed in the given environment. +For example, to run the test function that have the string "cover" in them +using pypy3, execute:: + + tox -e pypy3 -- pytest -k cover + +Or to get a Python prompt under Python 3.5 (with IMDbPY and all dependencies +already installed), execute:: + + tox -e py35 -- python + + +S3 dataset +---------- + +The tests will use the HTTP access system by default. If you would also like +to test the database generated from the S3 dataset, define the ``IMDBPY_S3_URI`` +environment variable:: + + IMDBPY_S3_URI='postgres://imdb@localhost/imdb' pytest + +This will run the tests for both HTTP and S3 access systems. + + +.. _pytest: https://pytest.org/ diff -pruN 5.1-1/docs/devel/translate.rst 6.6-1/docs/devel/translate.rst --- 5.1-1/docs/devel/translate.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/devel/translate.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,33 @@ +.. _translate: + +How to translate +---------------- + +.. note:: + + You can (but you don't have to) use Transifex to manage/coordinate + your translations: http://www.transifex.net/projects/p/imdbpy/ + +The :mod:`imdb.locale` package contains some scripts that are useful +for building your own internationalization files: + +- The :file:`generatepot.py` script should be used only when the DTD + is changed; it's used to create the :file:`imdbpy.pot` file + (the one that gets shipped is always up-to-date). + +- You can copy the :file:`imdbpy.pot` file as your language's ``.po`` file + (for example :file:`imdbpy-fr.po` for French) and modify it according + to your language. + +- Then you have to run the :file:`rebuildmo.py` script (which is automatically + executed at install time) to create the ``.mo`` files. + +If you need to upgrade an existing translation, after changes to the ``.pot`` +file (usually because the DTD was changed), you can use the ``msgmerge`` +utility which is part of the GNU gettext suite:: + + msgmerge -N imdbpy-fr.po imdbpy.pot > new-imdbpy-fr.po + +If you create a new translation or update an existing one, you can send +it to the mailing list, for inclusion +in upcoming releases. diff -pruN 5.1-1/docs/disclaimer.rst 6.6-1/docs/disclaimer.rst --- 5.1-1/docs/disclaimer.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/disclaimer.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,11 @@ +IMDbPY and its authors are not affiliated with Internet Movie Database Inc. + +IMDb is a trademark of Internet Movie Database Inc., and all content and data +included on the IMDb's site is the property of IMDb or its content suppliers +and protected by United States and international copyright laws. + +Please read the IMDb's conditions of use on their website: + +- https://www.imdb.com/conditions +- https://www.imdb.com/licensing +- any other notice on the https://www.imdb.com/ site diff -pruN 5.1-1/docs/DISCLAIMER.txt 6.6-1/docs/DISCLAIMER.txt --- 5.1-1/docs/DISCLAIMER.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/DISCLAIMER.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,15 +0,0 @@ - DISCLAIMER - ========== - -IMDbPY (and the author) is not affiliated with Internet Movie Database Inc. - -IMDb is a trademark of Internet Movie Database Inc. and all contents -and data included on the IMDb's site is the property of IMDb or its -content suppliers and protected by United States and international -copyright laws. - -Please, read the IMDb's conditions of use in their website: - - http://www.imdb.com/help/show_article?conditions - - http://www.imdb.com/help/show_leaf?usedatasoftware - - any other notice in the http://www.imdb.com/ site. - diff -pruN 5.1-1/docs/faqs.rst 6.6-1/docs/faqs.rst --- 5.1-1/docs/faqs.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/faqs.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,129 @@ +FAQs +==== + +:Q: Is IMDbPY compatible with Python 3? + +:A: Yes. Actually, the versions after 6.0 are compatible only with Python 3. + If you need an older, unmaintained, version for Python, see the + imdbpy-legacy branch in the repository. + + +:Q: Why is the movieID (and other IDs) used in the old "sql" database not + the same as the ID used on the IMDb.com site? + +:A: First, a bit of nomenclature: "movieID" is the term we use for a unique + identifier used by IMDbPY to manage a single movie (similar terms + for other kinds of data such as "personID" for persons). An "imdbID" + is the term we use for a unique identifier that the IMDb.com site uses + for the same kind of data (e.g.: the 7-digit number in tt0094226, + as seen in the URL for "The Untouchables"). + + When using IMDbPY to access the web ("http" data access system), movieIDs + and imdbIDs are the same thing -beware that in this case a movieID + is a string, with the leading zeroes. + + Unfortunately, when populating a SQL database with data from the plain text + data files, we don't have access to imdbIDs -since they are + not distributed at all- and so we have to generate them ourselves + (they are the "id" columns in tables like "title" or "name"). + This means that these values are valid only for your current database: + if you update it with a newer set of plain text data files, these IDs + will surely change (and, by the way, they are integers). + It's also obvious, now, that you can't exchange IDs between the "http" + and the "sql" data access systems, and similarly you can't use imdbIDs + with your local database or vice-versa. + + +:Q: When using a SQL database, what's the "imdb_id" (or something like that) + column in tables like "title", "name" and so on? + +:A: It's internally used by IMDbPY to remember the imdbID of a movie + (the one used by the web site), once it has been encountered. This way, + if IMDbPY is asked again about the imdbID of a movie (or person, or ...), + it won't have to contact the web site again. + + When accessing the database, you'll use the numeric value + of the "id" column, e.g. "movieID". Note that, to update the SQL database, + you have to access it using a user who has write permission. + + As a bonus, when possible, the values of the imdbIDs are saved + between updates of the SQL database (using the imdbpy2sql.py script). + Beware that it's tricky and not always possible, but the script does + its best to succeed. + + +:Q: But what if I really need the imdbIDs, to use in my database? + +:A: No, you don't. Search for a title, get its information. Be happy! + + +:Q: I have a great idea: Write a script to fetch all the imdbIDs + from the web site! Can't you do it? + +:A: Yeah, I can. But I won't. :-) + + It would be quite easy to map every title on the web to its imdbID, + but there are still lots of problems. First of all, every user will end up + doing it for their own copy of the plain text data files (and this will + make the imdbpy2sql.py script painfully slow and prone to all sort + of problems). Moreover, the imdbIDs are unique and never reused, true, + but movie titles _do_ change: to fix typos, to override working titles, + to cope with a new movie with the same title release in the same year, + not to mention cancelled or postponed movies. + + Other than that, we'd have to do the same for persons, characters, and + companies. Believe me: it doesn't make sense. Work on your local database + using your movieIDs (or even better: don't mind about movieIDs and think + in terms of searches and Movie instances!) and retrieve the imdbID only + in the rare circumstances when you really need them (see the next FAQ). + Repeat after me: I DON'T NEED ALL THE imdbIDs. :-) + + +:Q: When using a SQL database, how can I convert a movieID (whose value + is valid only locally) to an imdbID (the ID used by the imdb.com site)? + +:A: Various functions can be used to convert a movieID (or personID or + other IDs) to the imdbID used by the web site. Example: + + .. code-block:: python + + from imdb import IMDb + ia = IMDb('sql', uri=URI_TO_YOUR_SQL_DATABASE) + movie = ia.search_movie('The Untouchables')[0] # a Movie instance. + print('The movieID for The Untouchables:', movie.movieID) + print('The imdbID used by the site:', ia.get_imdbMovieID(movie.movieID)) + print('Same ID, smarter function:', ia.get_imdbID(movie)) + + It goes without saying that ``get_imdbMovieID`` method has some sibling + methods: ``get_imdbPersonID``, ``get_imdbCompanyID`` and + ``get_imdbCharacterID``. Also notice that the ``get_imdbID`` method + is smarter, and takes any kind of instance (the other functions need + a movieID, personID, ...) + + Another method that will try to retrieve the imdbID is ``get_imdbURL``, + which works like ``get_imdbID`` but returns a URL. + + In case of problems, these methods will return None. + + +:Q: I have a movie title (in the format used by the plain text data files) + or other kind of data (like a person/character/company name) and I want + to get its imdbID. How can I do it? + +:A: The safest thing is probably to do a normal search on IMDb (using + the "http" data access system of IMDbPY) and see if the first item is + the correct one. You can also try the "title2imdbID" method (and similar) + of the IMDb instance (no matter if you're using "http" or "sql"), but + expect some failures -in which case it will return None. + + +:Q: I have an URL (of a movie, person or something else), how can I + get a Movie/Person/... instance? + +:A: Import the ``imdb.helpers`` module and use the ``get_byURL`` function. + + +:Q: I'm writing an interface based on IMDbPY and I have problems handling + encoding, chars conversions, replacements of references and so on. + +:A: See the many functions in the imdb.helpers module. diff -pruN 5.1-1/docs/FAQS.txt 6.6-1/docs/FAQS.txt --- 5.1-1/docs/FAQS.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/FAQS.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,132 +0,0 @@ - IMDbPY FAQS - =========== - -Q1: Since version 3.7, parsing the data from the IMDb web site is slow, - sloow, slooow! Why? - -A1: if python-lxml is not installed in your system, IMDbPY uses the - pure-python BeautifulSoup module as a fall-back; BeautifulSoup does - an impressive job, but it can't be as fast as a parser written in C. - You can install python-lxml following the instructions in the - README.newparsers file. - - -Q2: why the movieID (and other IDs) used in the 'sql' database are not - the same used on the IMDb.com site? - -A2: first, a bit of nomenclature: we'll call "movieID" (or things like - "personID", for instance of the Person class) a unique identifier used - by IMDbPY to manage a single movie (or other kinds of object). - We'll call "imdbID" a unique identifier used, for the same kind - of data, by the IMDb.com site (i.e.: the 7-digit number in tt0094226, - as seen in the URL for "The Untouchables"). - - Using IMDbPY to access the web ('http' and 'mobile' data access - systems), movieIDs and imdbIDs are the same thing - beware that - in this case a movieID is a string, with the leading zeroes. - - Unfortunately, populating a sql database with data from the plain - text data files, we don't have access to imdbIDs - since they are - not distributed at all - and so we have to made them by ourselves - (they are the 'id' column in tables like 'title' or 'name'). - This mean that these values are valid only for your current database: - if you update it with a newer set of plain text data files, these IDs - will surely change (and, by the way, they are integers). - It's also obvious, now, that you can't exchange IDs between the - 'http' (or 'mobile') data access system and 'sql', and in the same - way you can't use imdbIDs with your local database or vice-versa. - - -Q3: using a sql database, what's the imdb_id (or something like that) - column in tables like 'title', 'name' and so on? - -A3: it's internally used by IMDbPY to remember the imdbID (the one - used by the web site - accessing the database you'll use the numeric - value of the 'id' column, as movieID) of a movie, once it stumbled - upon. This way, if IMDbPY is asked again about the imdbID of - a movie (or person, or ...), it doesn't have to contact again to - the web site. Notice that you have to access the sql database using - a user with write permission, to update it. - - As a bonus, when possible, the values of these imdbIDs are saved - between updates of the sql database (using the imdbpy2sql.py script). - Beware that it's tricky and not always possible, but the script does - its best to succeed. - - -Q4: but what if I really need the imdbIDs, to use my database? - -A4: no, you don't. Search for a title, get its information. Be happy! - - -Q5: I have a great idea: write a script to fetch all the imdbID from the - web site! Can't you do it? - -A5: yeah, I can. But I won't. :-) - It would be somewhat easy to map every title on the web to its - imdbID, but there are still a lot of problems. - First of all, every user will end up doing it for its own copy - of the plain text data files (and this will make the imdbpy2sql.py - script painfully slow and prone to all sort of problems). - Moreover, the imdbIDs are unique and never reused, true, but movie - title _do_ change: to fix typos, override working titles, to cope - with a new movie with the same title release in the same year (not - to mention cancelled or postponed movies). - - Besides that, we'd have to do the same for persons, characters and - companies. Believe me: it doesn't make sense. - Work on your local database using your movieIDs (or even better: - don't mind about movieIDs and think in terms of searches and Movie - instances!) and retrieve the imdbID only in the rare circumstances - when you really need them (see the next FAQ). - Repeat with me: I DON'T NEED ALL THE imdbIDs. :-) - - -Q6: using a sql database, how can I convert a movieID (whose value - is valid only locally) to an imdbID (the ID used by the imdb.com site)? - -A6: various functions can be used to convert a movieID (or personID or - other IDs) to the imdbID used by the web site. - Example of code: - - from imdb import IMDb - ia = IMDb('sql', uri=URI_TO_YOUR_SQL_DATABASE) - movie = ia.search_movie('The Untouchables')[0] # a Movie instance. - print 'The movieID for The Untouchables:', movie.movieID - print 'The imdbID used by the site:', ia.get_imdbMovieID(movie.movieID) - print 'Same ID, smarter function:', ia.get_imdbID(movie) - - It goes without saying that get_imdbMovieID has some sibling - methods: get_imdbPersonID, get_imdbCompanyID and get_imdbCharacterID. - Also notice that the get_imdbID method is smarter, and takes any kind - of instance (the other functions need a movieID, personID, ...) - - Another method that will try to retrieve the imdbID is get_imdbURL, - which works like get_imdbID but returns an URL. - - In case of problems, these methods will return None. - - -Q7: I have a movie title (in the format used by the plain text data files) - or other kind of data (like a person/character/company name) and I want - to get its imdbID. How can I do? - -A7: the safest thing, is probably to do a normal search on IMDb (using the - 'http' or 'mobile' data access system of IMDbPY) and see if the first - item is the correct one. - You can also try the 'title2imdbID' method (and similar) of the IMDb - instance (no matter if you're using 'http', 'mobile' or 'sql'), but - expect some failures - it returns None in this case. - - -Q8: I have an URL (of a movie, person or something else); how can I - get a Movie/Person/... instance? - -A8: import the imdb.helpers module and use the get_byURL function. - - -Q9: I'm writing an interface based on IMDbPY and I have problems handling - encoding, chars conversions, replacements of references and so on. - -A9: see the many functions in the imdb.helpers module. - diff -pruN 5.1-1/docs/goodies/download_applydiffs.py 6.6-1/docs/goodies/download_applydiffs.py --- 5.1-1/docs/goodies/download_applydiffs.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/goodies/download_applydiffs.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: UTF-8 -*- # This script downloads and applies any and all imdb diff files which @@ -33,16 +33,13 @@ # import os -import sys import shutil import subprocess import re import datetime -import time -import MySQLdb import logging -from datetime import timedelta,datetime +from datetime import timedelta from ftplib import FTP from random import choice @@ -137,7 +134,7 @@ def mktree(path): for path in paths_to_create: try: os.mkdir(path) - except Exception, e: + except Exception: logger.exception("Error trying to create %p" % path) return -1 return 0 @@ -173,18 +170,18 @@ def applyDiffs(): # # An even more robust approach would be to look inside each zipfile and read the date/time stamp # from the first line of the imdb list file itself but that seems like overkill to me. - day = None; + day = None for f in os.listdir(ImdbListsPath): if re.match(".*\.list\.gz",f) or re.match(".*\.list",f): try: t = os.path.getmtime(os.path.join(ImdbListsPath,f)) d = datetime.fromtimestamp(t) - if day == None: + if day is None: day = d elif d > day: day = d - except Exception, e: + except Exception as e: logger.exception("Unable to read last modified date for file %s" % f) if day is None: @@ -207,7 +204,7 @@ def applyDiffs(): if not os.path.isdir(ImdbDiffsPath): try: os.mkdir(ImdbDiffsPath) - except Exception, e: + except Exception as e: logger.exception("Unable to create folder for imdb diff files (%s)" % ImdbDiffsPath) return @@ -217,7 +214,7 @@ def applyDiffs(): while 1: if diffFileDate >= mostrecentfriday: - break; + break diff = "diffs-%s.tar.gz" % diffFileDate.strftime("%y%m%d") diffFilePath = os.path.join(ImdbDiffsPath, diff) @@ -225,7 +222,7 @@ def applyDiffs(): logger.debug("Need diff file %s" % diff) if not os.path.isfile(diffFilePath): - + # diff file is missing so we need to download it so first make sure we have an FTP connection if not haveFTPConnection: try: @@ -242,17 +239,17 @@ def applyDiffs(): ftp.cwd(ImdbDiffsFtpPath) haveFTPConnection = True - except Exception, e: + except Exception as e: logger.exception("Unable to connect to FTP server %s" % ImdbDiffsFtp) return # Now download the diffs file logger.info("Downloading ftp://%s%s/%s" % ( ImdbDiffsFtp, ImdbDiffsFtpPath, diff )) - diffFile = open(diffFilePath, 'wb'); + diffFile = open(diffFilePath, 'wb') try: ftp.retrbinary("RETR " + diff, diffFile.write) diffFile.close() - except Exception, e: + except Exception as e: # Unable to download diff file. This may be because it's not yet available but is due for release today code, message = e.message.split(' ', 1) @@ -292,20 +289,20 @@ def applyDiffs(): deleteFolder(tmpListsPath) try: os.mkdir(tmpListsPath) - except Exception, e: + except Exception as e: logger.exception("Unable to create temporary folder for imdb lists") return logger.info("Uncompressing imdb list files") # Uncompress list files in ImdbListsPath to our temporary folder tmpListsPath - numListFiles = 0; + numListFiles = 0 for f in os.listdir(ImdbListsPath): if re.match(".*\.list\.gz",f): try: cmdUnGzip = unGzip % (os.path.join(ImdbListsPath,f), tmpListsPath) subprocess.call(cmdUnGzip , shell=True) - except Exception, e: + except Exception as e: logger.exception("Unable to uncompress imdb list file using: %s" % cmdUnGzip) numListFiles += 1 @@ -320,7 +317,7 @@ def applyDiffs(): while 1: if imdbListsDate >= mostrecentfriday: - break; + break diff = "diffs-%s.tar.gz" % imdbListsDate.strftime("%y%m%d") diffFilePath = os.path.join(ImdbDiffsPath, diff) @@ -340,7 +337,7 @@ def applyDiffs(): try: cmdUnGzip = unGzip % (diffFilePath, tmpDiffsPath) subprocess.call(cmdUnGzip, shell=True) - except Exception, e: + except Exception as e: logger.exception("Unable to unzip imdb diffs file using: %s" % cmdUnGzip) return @@ -351,14 +348,14 @@ def applyDiffs(): try: cmdUnTar = unTar % (tarFile, tmpDiffsPath) subprocess.call(cmdUnTar, shell=True) - except Exception, e: + except Exception as e: logger.exception("Unable to untar imdb diffs file using: %s" % cmdUnTar) return # Clean up tar file and the sub-folder which 7z may have (weirdly) created while unTarring it - os.remove(tarFile); + os.remove(tarFile) if os.path.exists(os.path.join(tmpDiffsPath,"diffs")): - os.rmdir(os.path.join(tmpDiffsPath,"diffs")); + os.rmdir(os.path.join(tmpDiffsPath,"diffs")) # Apply all the patch files to the list files in tmpListsPath isFirstPatchFile = True @@ -368,11 +365,11 @@ def applyDiffs(): try: cmdApplyPatch = applyPatch % (os.path.join(tmpListsPath,f), os.path.join(tmpDiffsPath,f)) patchStatus = subprocess.call(cmdApplyPatch, shell=True) - except Exception, e: + except Exception as e: logger.exception("Unable to patch imdb list file using: %s" % cmdApplyPatch) patchStatus=-1 - if patchStatus <> 0: + if patchStatus != 0: # Patch failed so... logger.critical("Patch status %s: Wrong diff file for these imdb lists (%s)" % (patchStatus, diff)) @@ -383,7 +380,7 @@ def applyDiffs(): # Clean up temporary diff files deleteFolder(tmpDiffsPath) - if patchedOKWith <> None and isFirstPatchFile: + if patchedOKWith is not None and isFirstPatchFile: # The previous imdb diffs file succeeded and the current diffs file failed with the # first attempted patch, so we can keep our updated list files up to this point @@ -424,7 +421,7 @@ def applyDiffs(): if not os.path.isfile(os.path.join(diffFilesBackupFolder,diff)): try: shutil.copy(diffFilePath,diffFilesBackupFolder) - except Exception, e: + except Exception as e: logger.exception("Unable to copy %s to backup folder %s" % (diffFilePath, diffFilesBackupFolder)) if not keepDiffFiles: keepDiffFiles = True @@ -444,7 +441,7 @@ def applyDiffs(): try: cmdGZip = progGZip % os.path.join(tmpListsPath,f) subprocess.call(cmdGZip, shell=True) - except Exception, e: + except Exception as e: logger.exception("Unable to Gzip imdb list file using: %s" % cmdGZip) break if os.path.isfile(os.path.join(tmpListsPath,f)): @@ -453,7 +450,7 @@ def applyDiffs(): # Now move the updated and compressed lists to the main lists folder, replacing the old list files for f in os.listdir(tmpListsPath): if re.match(".*\.list.gz",f): - # Delete the original compressed list file from ImdbListsPath if it exists + # Delete the original compressed list file from ImdbListsPath if it exists if os.path.isfile(os.path.join(ImdbListsPath,f)): os.remove(os.path.join(ImdbListsPath,f)) @@ -476,9 +473,9 @@ def applyDiffs(): # If the imdb lists were successfully updated, even partially, then run my # DOS batch file "Update db from imdb lists.bat" to rebuild the imdbPy database # and relink and reintegrate my shadow tables data into it - if patchedOKWith <> None: + if patchedOKWith is not None: logger.info("imdb lists are updated up to imdb diffs file %s" % patchedOKWith) - if RunAfterSuccessfulUpdate <> None: + if RunAfterSuccessfulUpdate is not None: logger.info("Now running %s" % RunAfterSuccessfulUpdate) subprocess.call(RunAfterSuccessfulUpdate, shell=True) diff -pruN 5.1-1/docs/goodies/download-from-s3 6.6-1/docs/goodies/download-from-s3 --- 5.1-1/docs/goodies/download-from-s3 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/goodies/download-from-s3 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,6 @@ +#!/bin/sh + +today="`date +'%Y-%m-%d'`" + +wget --mirror --no-parent -A tsv.gz --no-host-directories --directory-prefix=imdb-dataset-${today} https://datasets.imdbws.com/ + diff -pruN 5.1-1/docs/goodies/README.txt 6.6-1/docs/goodies/README.txt --- 5.1-1/docs/goodies/README.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/goodies/README.txt 2018-08-05 13:36:02.000000000 +0000 @@ -5,6 +5,10 @@ Useful shell scripts, especially for dev See the comments at the top of the files for usage and configuration options. +download-from-s3: download the new alternative interface dataset. + +s3-reduce: create smaller versions of .tsv.gz files. + applydiffs.sh: Bash script useful apply patches to a set of IMDb's plain text data files. You can use this script to apply the diffs files distributed diff -pruN 5.1-1/docs/goodies/s3-reduce 6.6-1/docs/goodies/s3-reduce --- 5.1-1/docs/goodies/s3-reduce 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/goodies/s3-reduce 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,36 @@ +#!/bin/sh +# Copyright 2018 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# s3-reduce.sh: create smaller versions of .tsv.gz files + +COUNT="100" + +if [ "x$1" != "x" ] ; then + pushd "$1" +fi + +mkdir -p partials + +for fname in *.tsv.gz +do + zcat "${fname}" | head -${COUNT} | gzip -f - > "partials/${fname}" +done + + +if [ "x$1" != "x" ] ; then + popd +fi diff -pruN 5.1-1/docs/imdbpy.cfg 6.6-1/docs/imdbpy.cfg --- 5.1-1/docs/imdbpy.cfg 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/imdbpy.cfg 2018-08-05 13:36:02.000000000 +0000 @@ -39,7 +39,6 @@ accessSystem = http # Re-raise all caught exceptions (off, by default). #reraiseExceptions = off -## Optional (options common to http and mobile data access systems): # Proxy used to access the network. If it requires authentication, # try with: http://username:password@server_address:port/ #proxy = http://localhost:8080/ @@ -49,22 +48,7 @@ accessSystem = http ## Timeout for the connection to IMDb (30 seconds, by default). #timeout = 30 # Base url to access pages on the IMDb.com web server. -#imdbURL_base = http://akas.imdb.com/ - -## Parameters for the 'http' data access system. -# Parser to use; can be a single value or a list of value separated by -# a comma, to express order preference. Valid values: "lxml", "beautifulsoup" -#useModule = lxml,beautifulsoup - -## Parameters for the 'mobile' data access system. -#accessSystem = mobile - -## Parameters for the 'sql' data access system. -#accessSystem = sql -#uri = mysql://user:password@localhost/imdb -# ORM to use; can be a single value or a list of value separated by -# a comma, to express order preference. Valid values: "sqlobject", "sqlalchemy" -#useORM = sqlobject,sqlalchemy +#imdbURL_base = http://www.imdb.com/ ## Set the threshold for logging messages. # Can be one of "debug", "info", "warning", "error", "critical" (default: @@ -75,4 +59,3 @@ accessSystem = http # see: http://docs.python.org/library/logging.html#configuring-logging #loggingConfig = ~/.imdbpy-logger.cfg - diff -pruN 5.1-1/docs/index.rst 6.6-1/docs/index.rst --- 5.1-1/docs/index.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/index.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,28 @@ +IMDbPY +====== + +.. include:: ../README.rst + + +.. admonition:: Disclaimer + + .. include:: disclaimer.rst + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + usage/index + devel/index + faqs + contributors/index + Changelog + + +Indices and tables +================== + +- :ref:`genindex` +- :ref:`modindex` +- :ref:`search` diff -pruN 5.1-1/docs/INSTALL.txt 6.6-1/docs/INSTALL.txt --- 5.1-1/docs/INSTALL.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/INSTALL.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ - INSTALLATION - ============ - -See the "README.txt" file. - -You've to read it anyway, isn't it? - - diff -pruN 5.1-1/docs/LICENSE.txt 6.6-1/docs/LICENSE.txt --- 5.1-1/docs/LICENSE.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/LICENSE.txt 2018-08-05 13:36:02.000000000 +0000 @@ -1,17 +1,11 @@ -IMDbPY +# IMDbPY NOTE: see also the recommendations in the "DISCLAIMER.txt" file. -NOTE: for a list of other persons who share with me the copyright over +NOTE: for a list of persons who share the copyright over specific portions of code, see the "CONTRIBUTORS.txt" file. -NOTE: IMDbPY includes an unmodified version of BeautifulSoup, - renamed _bsoup.py; that code is copyrighted by its author, - Leonard Richardson and is released - under a New-style BSD license. - - - Copyright 2004-2009 Davide Alberani + Copyright 2004-2017 Davide Alberani et al. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff -pruN 5.1-1/docs/make.bat 6.6-1/docs/make.bat --- 5.1-1/docs/make.bat 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/make.bat 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=IMDbPY + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff -pruN 5.1-1/docs/Makefile 6.6-1/docs/Makefile --- 5.1-1/docs/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/Makefile 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = IMDbPY +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff -pruN 5.1-1/docs/modules/Character.rst 6.6-1/docs/modules/Character.rst --- 5.1-1/docs/modules/Character.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/Character.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.Character` +===================== + +.. automodule:: imdb.Character + :members: diff -pruN 5.1-1/docs/modules/cli.rst 6.6-1/docs/modules/cli.rst --- 5.1-1/docs/modules/cli.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/cli.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.cli` +=============== + +.. automodule:: imdb.cli + :members: diff -pruN 5.1-1/docs/modules/Company.rst 6.6-1/docs/modules/Company.rst --- 5.1-1/docs/modules/Company.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/Company.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.Company` +=================== + +.. automodule:: imdb.Company + :members: diff -pruN 5.1-1/docs/modules/_exceptions.rst 6.6-1/docs/modules/_exceptions.rst --- 5.1-1/docs/modules/_exceptions.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/_exceptions.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb._exceptions` +======================= + +.. automodule:: imdb._exceptions + :members: diff -pruN 5.1-1/docs/modules/helpers.rst 6.6-1/docs/modules/helpers.rst --- 5.1-1/docs/modules/helpers.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/helpers.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.helpers` +=================== + +.. automodule:: imdb.helpers + :members: diff -pruN 5.1-1/docs/modules/imdb.rst 6.6-1/docs/modules/imdb.rst --- 5.1-1/docs/modules/imdb.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/imdb.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb` +=========== + +.. automodule:: imdb + :members: diff -pruN 5.1-1/docs/modules/linguistics.rst 6.6-1/docs/modules/linguistics.rst --- 5.1-1/docs/modules/linguistics.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/linguistics.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.linguistics` +======================= + +.. automodule:: imdb.linguistics + :members: diff -pruN 5.1-1/docs/modules/locale.rst 6.6-1/docs/modules/locale.rst --- 5.1-1/docs/modules/locale.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/locale.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.locale` +================== + +.. automodule:: imdb.locale + :members: diff -pruN 5.1-1/docs/modules/_logging.rst 6.6-1/docs/modules/_logging.rst --- 5.1-1/docs/modules/_logging.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/_logging.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb._logging` +==================== + +.. automodule:: imdb._logging + :members: diff -pruN 5.1-1/docs/modules/Movie.rst 6.6-1/docs/modules/Movie.rst --- 5.1-1/docs/modules/Movie.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/Movie.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.Movie` +================= + +.. automodule:: imdb.Movie + :members: diff -pruN 5.1-1/docs/modules/parser.http.companyParser.rst 6.6-1/docs/modules/parser.http.companyParser.rst --- 5.1-1/docs/modules/parser.http.companyParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.companyParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.companyParser` +===================================== + +.. automodule:: imdb.parser.http.companyParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.movieParser.rst 6.6-1/docs/modules/parser.http.movieParser.rst --- 5.1-1/docs/modules/parser.http.movieParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.movieParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.movieParser` +===================================== + +.. automodule:: imdb.parser.http.movieParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.personParser.rst 6.6-1/docs/modules/parser.http.personParser.rst --- 5.1-1/docs/modules/parser.http.personParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.personParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.personParser` +==================================== + +.. automodule:: imdb.parser.http.personParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.rst 6.6-1/docs/modules/parser.http.rst --- 5.1-1/docs/modules/parser.http.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http` +======================= + +.. automodule:: imdb.parser.http + :members: diff -pruN 5.1-1/docs/modules/parser.http.searchCompanyParser.rst 6.6-1/docs/modules/parser.http.searchCompanyParser.rst --- 5.1-1/docs/modules/parser.http.searchCompanyParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.searchCompanyParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.searchCompanyParser` +=========================================== + +.. automodule:: imdb.parser.http.searchCompanyParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.searchKeywordParser.rst 6.6-1/docs/modules/parser.http.searchKeywordParser.rst --- 5.1-1/docs/modules/parser.http.searchKeywordParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.searchKeywordParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.searchKeywordParser` +=========================================== + +.. automodule:: imdb.parser.http.searchKeywordParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.searchMovieParser.rst 6.6-1/docs/modules/parser.http.searchMovieParser.rst --- 5.1-1/docs/modules/parser.http.searchMovieParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.searchMovieParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.searchMovieParser` +========================================= + +.. automodule:: imdb.parser.http.searchMovieParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.searchPersonParser.rst 6.6-1/docs/modules/parser.http.searchPersonParser.rst --- 5.1-1/docs/modules/parser.http.searchPersonParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.searchPersonParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.searchPersonParser` +========================================== + +.. automodule:: imdb.parser.http.searchPersonParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.topBottomParser.rst 6.6-1/docs/modules/parser.http.topBottomParser.rst --- 5.1-1/docs/modules/parser.http.topBottomParser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.topBottomParser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.topBottomParser` +======================================= + +.. automodule:: imdb.parser.http.topBottomParser + :members: diff -pruN 5.1-1/docs/modules/parser.http.utils.rst 6.6-1/docs/modules/parser.http.utils.rst --- 5.1-1/docs/modules/parser.http.utils.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.http.utils.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.http.utils` +============================= + +.. automodule:: imdb.parser.http.utils + :members: diff -pruN 5.1-1/docs/modules/parser.rst 6.6-1/docs/modules/parser.rst --- 5.1-1/docs/modules/parser.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,6 @@ +:orphan: + +:mod:`imdb.parser` +================== + +.. automodule:: imdb.parser diff -pruN 5.1-1/docs/modules/parser.s3.rst 6.6-1/docs/modules/parser.s3.rst --- 5.1-1/docs/modules/parser.s3.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.s3.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.s3` +===================== + +.. automodule:: imdb.parser.s3 + :members: diff -pruN 5.1-1/docs/modules/parser.s3.utils.rst 6.6-1/docs/modules/parser.s3.utils.rst --- 5.1-1/docs/modules/parser.s3.utils.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.s3.utils.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.s3.utils` +=========================== + +.. automodule:: imdb.parser.s3.utils + :members: diff -pruN 5.1-1/docs/modules/parser.sql.alchemyadapter.rst 6.6-1/docs/modules/parser.sql.alchemyadapter.rst --- 5.1-1/docs/modules/parser.sql.alchemyadapter.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.sql.alchemyadapter.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.sql.alchemyadapter` +===================================== + +.. automodule:: imdb.parser.sql.alchemyadapter + :members: diff -pruN 5.1-1/docs/modules/parser.sql.dbschema.rst 6.6-1/docs/modules/parser.sql.dbschema.rst --- 5.1-1/docs/modules/parser.sql.dbschema.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.sql.dbschema.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.sql.dbschema` +=============================== + +.. automodule:: imdb.parser.sql.dbschema + :members: diff -pruN 5.1-1/docs/modules/parser.sql.rst 6.6-1/docs/modules/parser.sql.rst --- 5.1-1/docs/modules/parser.sql.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/parser.sql.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.parser.sql` +====================== + +.. automodule:: imdb.parser.sql + :members: diff -pruN 5.1-1/docs/modules/Person.rst 6.6-1/docs/modules/Person.rst --- 5.1-1/docs/modules/Person.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/Person.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.Person` +================== + +.. automodule:: imdb.Person + :members: diff -pruN 5.1-1/docs/modules/utils.rst 6.6-1/docs/modules/utils.rst --- 5.1-1/docs/modules/utils.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/modules/utils.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,7 @@ +:orphan: + +:mod:`imdb.utils` +================= + +.. automodule:: imdb.utils + :members: diff -pruN 5.1-1/docs/README.adult 6.6-1/docs/README.adult --- 5.1-1/docs/README.adult 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.adult 1970-01-01 00:00:00.000000000 +0000 @@ -1,73 +0,0 @@ - IMDbPY for (too) sensitive people - ================================= - -Since version 2.0 (shame on me! I've noticed this only after more -than a year of development!!!) by default adult movies are included -in the result of the search_movie(), search_episode() and search_person() -methods. - -If for some unintelligible reason you don't want classics -like "Debbie Does Dallas" to show up in your list of results, -you can disable this feature initializing the IMDb class with -the 'adultSearch' argument set to 0 (or other "False" value). - -E.g.: - from imdb import IMDb - ia = IMDb(accessSystem='http', adultSearch=0) - - -The behavior of a IMDb class's instance can be modified at -runtime, calling the do_adult_search() method. - -E.g.: - from imdb import IMDb - - # By default in the horny-mode. - ia = IMDb(accessSystem='http') - - # Just for this example, be sure to exclude the proxy. - ia.set_proxy(None) - - results = ia.search_movie('debby does dallas', results=5) - for movie in results: - print movie['long imdb title'], movie.movieID - # It will print: - # Debbie Does Dallas (1978) 0077415 - # Debbie Does Dallas Part II (1981) 0083807 - # Debbie Does Dallas: The Next Generation (1997) (V) 0160174 - # Debbie Does Dallas '99 (1999) (V) 0233539 - # Debbie Does Dallas 3 (1985) 0124352 - - # You can now revert to the old puritan behavior. - ia.do_adult_search(0) - - results = ia.search_movie('debby does dallas', results=5) - for movie in results: - print movie['long imdb title'], movie.movieID - # It will print only: - # Pauly Does Dallas (1993) (TV) 0208347 - - -The do_adult_search() method of the http and mobile data access system -also takes another couple of arguments: "cookie_id" and "cookie_uu", so -that you can select _your own_ IMDb's account; if cookie_id is set to -None, no cookies are sent. These parameters can also be set in -the imdbpy.cfg configuration file. -For the strings to use, see your "cookie" or "cookie.txt" file. -Obviously you need to activate the "adult movies" option for -your account; see http://imdb.com/find/preferences?_adult=1 - - - OTHER DATA ACCESS SYSTEMS - ========================= - -Since version 2.2 every other data access system (sql) -support the same behavior of the http and mobile data access -systems (i.e.: you can set the 'adultSearch' argument and use -the 'do_adult_search' method). - -Notice that for the sql data access system only results from the -search_movie() and search_episode() methods are filtered: there's no -easy (and fast) way to tell that an actor/actress is a porn-star. - - diff -pruN 5.1-1/docs/README.companies 6.6-1/docs/README.companies --- 5.1-1/docs/README.companies 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.companies 1970-01-01 00:00:00.000000000 +0000 @@ -1,18 +0,0 @@ - - COMPANIES DATA - ============== - -Since IMDbPY 3.6, companies have their own class and every reference -to a company is now an instance of the Company class. - - NOTES - ===== - -- no references (inside text field) about companies are retrieved, - so far; I don't even think there are any in the IMDb web site, but - I may be wrong. - -- 'mobile' data access system collects companies informations through - the 'httpThin' method; it should be fast enough. - - diff -pruN 5.1-1/docs/README.currentRole 6.6-1/docs/README.currentRole --- 5.1-1/docs/README.currentRole 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.currentRole 1970-01-01 00:00:00.000000000 +0000 @@ -1,97 +0,0 @@ - - THE currentRole ATTRIBUTE AND THE Character CLASS - ================================================= - -Since version 3.3, IMDbPY supports the character pages of the IMDb -database; this required some substantial changes to how actors' -and acresses' roles were handled. -Starting with release 3.4, "sql" data access system is supported, -too - but it works a bit differently from "http" and "mobile". -See "SQL" below. - -The currentRole instance attribute can be found in every instance -of Person, Movie and Character classes, even if actually the Character -never uses it. - -The currentRole of a Person object is set to a Character instance, -inside a list of person who acted in a given movie. -The currentRole of a Movie object is set to a Character instance, -inside a list of movies played be given person. -The currentRole of a Movie object is set to a Person instance, -inside a list of movies in which a given character was portrayed. - -Schema: - movie['cast'][0].currentRole -> a Character object. - | - +-> a Person object. - - person['actor'][0].currentRole -> a Character object. - | - +-> a Movie object. - - character['filmography'][0].currentRole -> a Person object. - | - +-> a Movie object. - -The roleID attribute can be used to access/set the characterID -or personID instance attribute of the current currentRole. -Building Movie or Person objects, you can pass the currentRole -parameter and the roleID parameter (to set the ID). -The currentRole parameter can be an object (Character or Person), -an unicode string (in which case a Character or Person object is -automatically instanced) or a list of objects or strings (to -handle multiple characters played by the same actor/actress in -a movie, or character played by more then a single actor/actress -in the same movie). - -Anyway, currentRole objects (Character or Person instances) can -be pretty-printed easily: calling unicode(CharacterOrPersonObject) -will return a good-old-unicode string, like expected in the previous -version of IMDbPY. - - - SQL - === - -Fetching data from the web, only characters with an active page -on the web site will have their characterID; we don't have these -information accessing "sql", so _every_ character will have an -associated characterID. -This way, every character with the same name will share the same -characterID, even if - in fact - they may not be portraying the -same character. - - - GOODIES - ======= - -To help getting the required information from Movie, Person and -Character objects, in the "helpers" module there's a new factory -function, makeObject2Txt, which can be used to create your -pretty-printing function. -It takes some optional parameters: movieTxt, personTxt, characterTxt -and companyTxt; in these strings %(value)s items are replaced with -object['value'] or with obj.value (if the first is not present). - -E.g.: - import imdb - myPrint = imdb.helpers.makeObject2Txt(personTxt=u'%(name)s ... %(currentRole)s') - i = imdb.IMDb() - m = i.get_movie('0057012') - ps = m['cast'][0] - print myPrint(ps) - # The output will be something like: - Peter Sellers ... Group Captain Lionel Mandrake / President Merkin Muffley / Dr. Strangelove - - -Portions of the formatting string can be stripped conditionally: if -the specified condition is false, they will be cancelled. - -E.g.: - myPrint = imdb.helpers.makeObject2Txt(personTxt='%(long imdb name)s ... %(currentRole)s %(notes)s' - - -Another useful argument is 'applyToValues': if set to a function, -it will be applied to every value before the substitution; it can -be useful to format strings for html output. - diff -pruN 5.1-1/docs/README.devel 6.6-1/docs/README.devel --- 5.1-1/docs/README.devel 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.devel 1970-01-01 00:00:00.000000000 +0000 @@ -1,274 +0,0 @@ - DEVELOPMENT OF IMDbPY - ===================== - -A lot of other information useful to IMDbPY developers are available -in the "README.package" file. - -Sections in this file: -* STRUCTURE OF THE IMDbPY PACKAGE -* GENERIC DESCRIPTION -* HOW TO EXTEND - - - STRUCTURE OF THE IMDbPY PACKAGE - =============================== - -imdb (package) - | - +-> _compat - +-> _exceptions - +-> _logging - +-> linguistics - +-> Movie - +-> Person - +-> Character - +-> Company - +-> utils - +-> helpers - +-> parser (package) - | - +-> http (package) - | | - | +-> movieParser - | +-> personParser - | +-> characterParser - | +-> companyParser - | +-> searchMovieParser - | +-> searchPersonParser - | +-> searchCharacterParser - | +-> searchCompanyParser - | +-> searchKeywordParser - | +-> topBottomParser - | +-> utils - | +-> bsouplxml - | | - | +-> _bsoup.py - | +-> etree.py - | +-> html.py - | +-> bsoupxpath.py - | - +-> mobile (package) - | - +-> sql (package) - | - +-> dbschema - +-> alchemyadapter - +-> objectadapter - +-> cutils (C module) - - -Description: -imdb (package): contains the IMDb function, the IMDbBase class and imports - the IMDbError exception class. -_compat: compatibility functions and class for some strange environments - (internally used). -_exceptions: defines the exceptions internally used. -_logging: provides the logging facility used by IMDbPY. -linguistics: defines some functions and data useful to smartly guess the - language of a movie title (internally used). -Movie: contains the Movie class, used to describe and manage a movie. -Person: contains the Person class, used to describe and manage a person. -Character: contains the Character class, used to describe and manage - a character. -Company: contains the Company, used to describe and manage a company. -utils: miscellaneous utilities used by many IMDbPY modules. -parser (package): a package containing a package for every data access system - implemented. -http (package): contains the IMDbHTTPAccessSystem class which is a subclass - of the imdb.IMDbBase class; it provides the methods used to - retrieve and manage data from the web server (using, - in turn, the other modules in the package). - It defines methods to get a movie and to search for a title. -http.movieParser: parse html strings from the pages on the IMDb web server about - a movie; returns dictionaries of {key: value} -http.personParser: parse html strings from the pages on the IMDb web server - about a person; returns dictionaries. -http.characterParser: parse html strings from the pages on the IMDb web server - about a character; returns dictionaries. -http.companyParser: parse html strings from the pages on the IMDb web server - about a company; returns dictionaries. -http.searchMovieParser: parse an html string, result of a query for a movie - title. -http.searchPersonParser: parse an html string, result of a query for a person - name. -http.searchCharacterParser: parse an html string, result of a query for a - character name. -http.searchCompanyParser: parse an html string, result of a query for a - company name. -http.searchKeywordParser: parse an html string, result of a query for a keyword. -http.topBottomParser: parse an html string, result of a query for top250 - and bottom100 movies. -http.utils: miscellaneous utilities used only by the http package. -http.bsouplxml (package): adapter to make BeautifulSoup behave like lxml - (internally, the API of lxml is always used). -http.bsouplxml._bsoup: just a copy of the BeautifulSoup module, so that it's not - an external dependency. -http.bsouplxml.etree: adapter for the lxml.etree module. -http.bsouplxml.html: adapter for the lxml.html module. -http.bsouplxml.bsoupxpath: xpath support for beautifulsoup. - - -The parser.sql package manages the access to the data in the SQL -database, created with the imdbpy2sql.py script; see the README.sqldb file. -The dbschema module contains tables definitions and some useful functions; -The alchemyadapter adapts the SQLAlchemy ORM to the internal mechanisms -of IMDbPY, and the objectadapter does the same for the SQLObject ORM -(internally the API of SQLObject is always used). -The cutils module is a C module containing C function to speed up the -'sql' data access system; if it can't be compiled, a set of fall'back -functions will be used. - -The class in the parser.mobile package is a subclass of the one found -in parser.http, with some method overridden to be many times faster (from -2 to 20 times); it's useful for systems with slow bandwidth and not -much CPU power. - -The helpers module contains functions and other goodies not directly -used by the IMDbPY package, but that can be useful to develop -IMDbPY-based programs. - - - GENERIC DESCRIPTION - =================== - -I wanted to stay independent from the source of the data for a given -movie/person/character/company, and so the imdb.IMDb function returns -an instance of a class that provides specific methods to access a given -data source (web server, SQL database, etc.) - -Unfortunately that means that the movieID in the Movie class, the -personID in the Person class and the characterID in the Character class -are dependent on the data access system used. -So, when a Movie, a Person or a Character object is instantiated, the -accessSystem instance variable is set to a string used to identify the -used data access system. - - - HOW TO EXTEND - ============= - -To introduce a new data access system, you've to write a new package -inside the "parser" package; this new package must provide a subclass -of the imdb.IMDb class which must define at least the following methods: - _search_movie(title) - to search for a given title; must return a - list of (movieID, {movieData}) tuples. - _search_episode(title) - to search for a given episode title; must return a - list of (movieID, {movieData}) tuples. - _search_person(name) - to search for a given name; must return a - list of (movieID, {personData}) tuples. - _search_character(name) - to search for a given character's name; must - return a list of (characterID, {characterData}) - tuples. - _search_company(name) - to search for a given company's name; must - return a list of (companyID, {companyData}) - tuples. - get_movie_*(movieID) - a set of methods, one for every set of information - defined for a Movie object; should return - a dictionary with the relative information. - This dictionary can contains some optional keys: - 'data': must be a dictionary with the movie info. - 'titlesRefs': a dictionary of 'movie title': movieObj - pairs. - 'namesRefs': a dictionary of 'person name': personObj - pairs. - get_person_*(personID) - a set of methods, one for every set of information - defined for a Person object; should return - a dictionary with the relative information. - get_character_*(characterID) - a set of methods, one for every set of - information defined for a character object; - should return a dictionary with the relative - information. - get_company_*(companyID) - a set of methods, one for every set of - information defined for a company object; - should return a dictionary with the relative - information. - _get_top_bottom_movies(kind) - kind can be one of 'top' and 'bottom'; - returns the related list of movies. - _get_keyword(keyword) - return a list of Movie objects with the given keyword. - _search_keyword(key) - return a list of keywords similar to the given key. - get_imdbMovieID(movieID) - must convert the given movieID to a string - representing the imdbID, as used by the IMDb web - server (e.g.: '0094226' for Brian De Palma's - "The Untouchables"). - get_imdbPersonID(personID) - must convert the given personID to a string - representing the imdbID, as used by the IMDb web - server (e.g.: '0000154' for "Mel Gibson"). - get_imdbCharacterID(characterID) - must convert the given characterID to a - string representing the imdbID, as used by - the IMDb web server (e.g.: '0000001' for - "Jesse James"). - get_imdbCompanyID(companyID) - must convert the given companyID to a - string representing the imdbID, as used by - the IMDb web server (e.g.: '0071509' for - "Columbia Pictures [us]"). - _normalize_movieID(movieID) - must convert the provided movieID in a - format suitable for internal use (e.g.: - convert a string to a long int). - NOTE: as a rule of thumb you _always_ need - to provide a way to convert a "string - representation of the movieID" into the - internally used format, and the internally - used format should _always_ be converted to - a string, in a way or another. - Rationale: a movieID can be passed from the - command line, or from a web browser. - _normalize_personID(personID) - idem. - _normalize_characterID(characterID) - idem. - _normalize_companyID(companyID) - idem. - _get_real_movieID(movieID) - return the true movieID; useful to handle - title aliases. - _get_real_personID(personID) - idem. - _get_real_characterID(characterID) - idem. - _get_real_companyID(companyID) - idem. - -The class should raise the appropriate exceptions, when needed; -IMDbDataAccessError must be raised when you cannot access the resource -you need to retrieve movie info or you're unable to do a query (this is -_not_ the case when a query returns zero matches: in this situation an -empty list must be returned); IMDbParserError should be raised when an -error occurred parsing some data. - -Now you've to modify the imdb.IMDb function so that, when the right -data access system is selected with the "accessSystem" parameter, an -instance of your newly created class is returned. - -NOTE: this is a somewhat misleading example: we already have a -data access system for sql database (it's called 'sql' and it supports -also MySQL, amongst other). Maybe I'll find a better example... -E.g.: if you want to call your new data access system "mysql" (meaning -that the data are stored in a mysql database), you've to add to the imdb.IMDb -function something like: - if accessSystem == 'mysql': - from parser.mysql import IMDbMysqlAccessSystem - return IMDbMysqlAccessSystem(*arguments, **keywords) - -where "parser.mysql" is the package you've created to access the -local installation, and "IMDbMysqlAccessSystem" is the subclass of -imdb.IMDbBase. -Then it's possibile to use the new data access system like: - from imdb import IMDb - i = IMDb(accessSystem='mysql') - results = i.search_movie('the matrix') - print results - -A specific data access system implementation can defines it's own -methods. -As an example, the IMDbHTTPAccessSystem that is in the parser.http package -defines the method set_proxy() to manage the use a web proxy; you -can use it this way: - from imdb import IMDb - i = IMDb(accessSystem='http') # the 'accessSystem' argument is not - # really needed, since "http" is the default. - i.set_proxy('http://localhost:8080/') - -A list of special methods provided by the imdb.IMDbBase subclass, along -with their description, is always available calling the get_special_methods() -of the IMDb class. -E.g.: - i = IMDb(accessSystem='http') - print i.get_special_methods() - -will print a dictionary with the format: - {'method_name': 'method_description', ...} - - diff -pruN 5.1-1/docs/README.http 6.6-1/docs/README.http --- 5.1-1/docs/README.http 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.http 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ - IMDbPY HTTP CONNECTION - ====================== - -HTTP is the default data access system of IMDbPY, meaning that by default -data are requested at the IMDb web servers. -For other kinds of data access, see README.sqldb and README.mobile. - -By default IMDbPY uses its own account to access the IMDb web server (this -is done to enable searches on adult titles); if you want to uses your own -account, see README.adult. - - CONNECTION PROBLEMS - =================== - -It has been reported some kind of problems connecting to the IMDb servers; -the problem seems to be related to the use of our cookie and the geographical -location of the user. -If you experience such a problem, report it and try to disable the use of the -cookie (to do so, see README.adult). - diff -pruN 5.1-1/docs/README.info2xml 6.6-1/docs/README.info2xml --- 5.1-1/docs/README.info2xml 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.info2xml 1970-01-01 00:00:00.000000000 +0000 @@ -1,100 +0,0 @@ - - INFORMATION IN XML FORMAT - ========================= - -Since version 4.0, IMDbPY can output information of Movie, Person, -Character and Company instances in XML format. -It's possible to get a single information (a key) in XML format, -using the getAsXML(key) method (it will return None if the key is -not found). -E.g.: - from imdb import IMDb - ia = IMDb('http') - movie = ia.get_movie(theMovieID) - print movie.getAsXML('keywords') - -It's also possible to get a representation of a whole object, -using the asXML() method: - print movie.asXML() - -The returned strings are unicode. The _with_add_keys argument -of the asXML() method can be set to False (default: True) to -exclude the dynamically generated keys (like 'smart canonical title' -and so on). - - - XML FORMAT - ========== - -Keywords are converted to tags, items in lists are enclosed in -a 'item' tag. E.g.: - - a keyword - another keyword - - -Except when keys are known to be not fixed (e.g.: a list of keywords), -in which case this schema is used: - - ... - - -In general, the 'key' attribute is present whenever the used tag -doesn't match the key name. - -Movie, Person, Character and Company instances are converted like -that (portions enclosed in squares are optionals): - - A Long IMDb Movie Title (YEAR) - [ - - Name Surname - [A Note About The Person] - - ] - [A Note About The Movie] - - -Every 'id' can be empty. - -Actually the returned XML is mostly not pretty-printed. - - - REFERENCES - ========== - -Some text keys can contain references to other movies, persons -and characters. The user can provide the defaultModFunct function (see -the "MOVIE TITLES AND PERSON/CHARACTER NAMES REFERENCES" section of -the README.package file), to replace these references with their own -strings (e.g.: a link to a web page); it's up to the user, to be sure -that the output of the defaultModFunct function is valid XML. - - - DTD - === - -Since version 4.1 a DTD is available; it can be found in this -directory or on the web, at: - http://imdbpy.sf.net/dtd/imdbpy41.dtd - -The version number changes with the IMDbPY version. - - - LOCALIZATION - ============ - -Since version 4.1 it's possible to translate the XML tags; -see README.locale. - - - FROM XML TO OBJECTS - =================== - -Since version 4.6, you can dump the generated XML in a string or -in a file, using it - later - to rebuild the original object. -In the imdb.helpers module there's the parseXML() function which -takes a string as input and return - if possible - an instance of -the Movie, Person, Character or Company classes. - - diff -pruN 5.1-1/docs/README.keywords 6.6-1/docs/README.keywords --- 5.1-1/docs/README.keywords 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.keywords 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ - - KEYWORDS - ======== - -Since version 4.0, it's possible (for every data access system) to -search for movies' keywords. -People's keywords are not supported. - - - SEARCH FOR A KEYWORD SIMILAR TO A GIVEN STRING - ============================================== - -The search_keyword(unicode_string) can be used to search amongst -keywords: a list of keywords similar to the given string will be -returned, sorted by similarity. Notice that the keywords in the -returned list are plain unicode strings, and not instances of -some class (like the ones returned by other search_SOMETHING methods). - -E.g.: - from imdb import IMDb - ia = IMDb('http') - print ia.search_keyword(u'alabama') - - - GET A LIST OF MOVIES FOR A GIVEN KEYWORD - ======================================== - -To get a list of movies that are tagged with the given keyword, -use the get_keyword(unicode_string) method. -E.g.: - from imdb import IMDb - ia = IMDb('http') - print ia.get_keyword(u'alabama') - -Beware that by default the list is limited to 100 movies, and -it's not possible to get more results, using 'http'. -Moreover, the lists returned using 'sql' are not sorted in any way. -Another limit is that actually (as of february 2009), the IMDb's -web server is unable to serve pages about non-ascii keywords. -It's a known problem of their systems. - diff -pruN 5.1-1/docs/README.local 6.6-1/docs/README.local --- 5.1-1/docs/README.local 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.local 1970-01-01 00:00:00.000000000 +0000 @@ -1,18 +0,0 @@ - LOCAL INSTALLATION - ================== - -Simple instruction: switch to 'sql' (see the README.sqldb file). - -The 'local' data access system was removed since IMDbPY 4.2, for -a series of good reasons: -- the moviedb program was no more distributed by IMDb. -- the new format for movie titles ("The Title" instead of "Title, The") - created way too many problems almost impossible to fix, since a lot - of damage was done by the - never updated - moviedb program. -- it was slower and less complete than 'sql'. -- there were very few users of it. - - -If you are veeery willing to resuscitate it, you can write in -the mailing list about your crazy idea. :-) - diff -pruN 5.1-1/docs/README.locale 6.6-1/docs/README.locale --- 5.1-1/docs/README.locale 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.locale 1970-01-01 00:00:00.000000000 +0000 @@ -1,109 +0,0 @@ - - LOCALIZATION FOR IMDbPY - ======================= - -Since version 4.1 it's easy to translate the labels that describe -sets of information. - - - LIMITATION - ========== - -So far no internal message or exception is translated, the -internationalization is limited to the "tags" returned -by the getAsXML and asXML methods of the Movie, Person, Character -or Company classes. Beware that in many cases these "tags" are not -the same as the "keys" used to access information in the same -classes, as if they are dictionaries. -E.g.: you can translate "long-imdb-name" - the tag returned by -the call person.getAsXML('long imdb name') - but not "long imdb name" -directly. -To translate keys, you can use the helpers.translateKey function in -the 'helpers' module. - - - USAGE - ===== - -If you want to add i18n to your IMDbPY-based application, all you need -to do is to switch to the 'imdbpy' text domain. - -E.g.: - import imdb.locale - - # Standard gettext stuff. - import gettext - from gettext import gettext as _ - - # Switch to the imdbpy domain. - gettext.textdomain('imdbpy') - - # Request a translation. - print _(u'long-imdb-name') - - - ADD A NEW LANGUAGE - ================== - -You can (but you're not forced to) use Transifex to manage/coordinate -your translations; see: http://www.transifex.net/projects/p/imdbpy/c/default/ -Below, the generic instruction about how translation works. - -In the imdb.locale package, you'll find some scripts useful to build -your own internationalization files. -If you create a new translation or update an existing one, you can send -it to the mailing list, for -inclusion in the next releases. - -- the generatepot.py should be used only when the DTD is changed; it's - used to create the imdbpy.pot file (the one shipped is always - up-to-date). -- you can copy the imdbpy.pot file to your language's .po file (e.g. - imdbpy-fr.po for French) and modify it accordingly to your needs. -- then you must run rebuildmo.py (which is automatically called - at install time, by the setup.py script) to create the .mo files. - -If you need to upgrade an existing .po file, after changes to the .pot -file (usually because the DTD was changed), you can use the msgmerge -tool, part of the GNU gettext suite. -E.g.: - msgmerge -N imdbpy-fr.po imdbpy.pot > new-imdbpy-fr.po - - - - ARTICLES IN TITLES - ================== - -Converting a title to its 'Title, The' canonical format, IMDbPY does -some assumptions about what is an article and what not, and this could -lead to some wrong canonical titles. E.g.: "Hard, Die" instead of -"Die Hard", since 'Die' is guessed as an article (and it is, in Germany...) -To solve this problem, there are other keys: "smart canonical title", -"smart long imdb canonical title", "smart canonical series title", -"smart canonical episode title" which can be used to do a better job -converting a title into its canonical format. - -It works, but it needs to know something about articles in various -languages: if you want to help, see the LANG_ARTICLES and LANG_COUNTRIES -dictionaries in the 'linguistics' module. - -To know what the language in which a movie title is assumed to be, -call its 'guessLanguage' method (it will return None, if unable to guess). -If you want to force a given language instead of the guessed one, you -can call its 'smartCanonicalTitle' method, setting the 'lang' argument -appropriately. - - - TITLE AKAS - ========== - -Sometimes it's useful to manage title's AKAs knowing their languages. -In the 'helpers' module there are some (hopefully) useful functions: -akasLanguages(movie) - given a movie, return a list of tuples - in (lang, AKA) format (lang can be None, if unable to detect). -sortAKAsBySimilarity(movie, title) - sorts the AKAs on a movie considering - how much they are similar to a given title (see - the code for more options). -getAKAsInLanguage(movie, lang) - return a list of AKAs of the movie in the given - language (see the code for more options). - diff -pruN 5.1-1/docs/README.logging 6.6-1/docs/README.logging --- 5.1-1/docs/README.logging 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.logging 1970-01-01 00:00:00.000000000 +0000 @@ -1,15 +0,0 @@ - - LOGGING - ======= - -Since version 4.4 IMDbPY provides a logging facility, using the -powerful "logging" module. -You can find documentation about it here: - http://docs.python.org/library/logging.html - -By default information are logged on standard error; you can read -on the module documentation how to stream them elsewhere. - -The default logging level is "warning"; this can be changed -modifying the "loggingLevel" key of your imdbpy.cfg file. - diff -pruN 5.1-1/docs/README.mobile 6.6-1/docs/README.mobile --- 5.1-1/docs/README.mobile 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.mobile 1970-01-01 00:00:00.000000000 +0000 @@ -1,114 +0,0 @@ - IMDbPY FOR SMALL SYSTEMS - ======================== - -Since version 1.8, IMDbPY tries to be usable even on -systems with very limited storage space, bandwidth and -CPU power, like PDA, hand-held devices and mobile phones. - -Sections in this file: -* INSTALLATION OPTIONS - how to save a little space installing IMDbPY. -* THE "MOBILE" DATA ACCESS SYSTEM - useful for systems with very little CPU power and bandwidth. -* THE "HTTPTHIN" DATA ACCESS SYSTEM - for systems with normal CPU power, but insufficient bandwidth. -* OTHER TIPS - -Please read all the following section. - - - INSTALLATION OPTIONS - ==================== - -You can call the setup.py script with some arguments: - -The --without-sql argument, if used, will excludes the parser.sql -package; you don't need it if your system does not have any of the -SQLObject or SQLAlchemy packages and/or you don't want to store the -whole IMDb's plain text database files in a SQL database. - -Now, if you're installing IMDbPY (using ./setup.py install), you -should take a look at some options, like "--no-compile" and "-O0" -to exclude pyc and pyo files, saving hundreds of KBs. - -Moreover, if you're creating a package (rpm, deb or whatever), -in the setup.cfg you can exclude from your package things -like the documentation (more than 200Kb) and the scripts in the -./bin/ directory. - - - THE "MOBILE" DATA ACCESS SYSTEM - =============================== - -Intended to be used with PDA, smart phones and hand-held devices, -the "mobile" data access system is a subclass of the default -"httpThin" data access system, with some methods replaced with faster -string methods, instead of the html parser. Moreover, for -the movies, only the main information are retrieved (see the 'httpThin' -notes). It should be, at usage time, from 2 to 20 times faster than -the "http"/"httpThin" data access system. - -This code still needs tests on mobile phones! -Please report any bugs/ideas/hints... - -Usage: - from imdb import IMDb - i = IMDb('mobile') - sp = i.search_person('mel gibson', results=10) - p = sp[0] - i.update(p) - sm = i.search_movie('mystic river', results=15) - m = sm[0] - i.update(m) - ...and so on... - - -A GUI for Series 60 smart phones, is available at: - http://imdbpy.sourceforge.net/?page=mobile - - - THE "HTTPTHIN" DATA ACCESS SYSTEM - ================================= - -Instead of the default data access system ('http'), you can -also use 'httpThin' (or 'webThin' or 'htmlThin'). - -I.e.: - from imdb import IMDb - i = IMDb('httpThin') - sp = i.search_person('mel gibson', results=10) - sm = i.search_movie('mystic river', results=15) - ...and so on... - - -The main difference is that, parsing movies' information, -the "maindetails" page is parsed, in place of the "combined" page. -This reduces the required bandwidth and the CPU power needed. -Obviously a lot of information are lost (and only the first 15 -people of the cast are listed), but it still retrieves everything -you usually need (director, writer, runtime, country, language, akas, -etc.) - -Another difference is that, if the "defaultModFuct" parameter is not -provided (as default) calling the IMDb() function, no references -to people or movie are collected from textual information (like -the plot of a movie). - - - OTHER TIPS - ========== - -Remember that, calling the search_movie(), search_episode() and -search_person() methods of the "IMDb" object, you can provide a "results" -parameter, to download only a limited amount of results (20, -by default). - -With the http, httpThin and mobile data access systems you can -set a proxy with the set_proxy() method; e.g.: - i = IMDb('http') - i.set_proxy('http://localhost:8080/') - -Remember that the proxy is automatically used if the $HTTP_PROXY -environment variable is set. - - diff -pruN 5.1-1/docs/README.newparsers 6.6-1/docs/README.newparsers --- 5.1-1/docs/README.newparsers 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.newparsers 1970-01-01 00:00:00.000000000 +0000 @@ -1,81 +0,0 @@ - IMDbPY'S NEW HTML PARSERS - ========================= - -Since version 3.7, IMDbPY has moved its parsers for the HTML of -the IMDb's website from a set of subclasses of SGMLParser (they -were finite-states machines, being SGMLParser a SAX parser) to -a set of parsers based on the libxml2 library or on the BeautifulSoup -module (and so, using a DOM/XPath-based approach). -The idea and the implementation of these new parsers is mostly a -work of H. Turgut Uyar, and can bring to parsers that are shorter, -easier to write and maybe even faster. - -The old set of parsers was removed since IMDbYP 4.0. - - - LIBXML AND/OR BEAUTIFULSOUP - =========================== - -To use "lxml", you need the libxml2 library installed (and its -python-lxml binding). If it's not present on your system, you'll -fall-back to BeautifulSoup - distributed alongside IMDbPY, and so -you don't need to install anything. -However, beware that being pure-Python, BeautifulSoup is much -slower than lxml, so install it, if you can. - -If for some reason you can't get lxml and BeautifulSoup is too -slow for your needs, consider the use of the 'mobile' data -access system. - - - GETTING LIBXML, LIBXSLT AND PYTHON-LXML - ======================================= - -If you're in a Microsoft Windows environment, all you need is -python-lxml (it includes all the required libraries), which can -be downloaded from here: - http://pypi.python.org/pypi/lxml/ - -Otherwise, if you're in a Unix environment, you can download libxml2 -and libxslt from here (you need both, to install python-lxml): - http://xmlsoft.org/downloads.html - http://xmlsoft.org/XSLT/downloads.html - -The python-lxml package can be found here: - http://codespeak.net/lxml/index.html#download - -Obviously you should first check if these libraries are already -packaged for your distribution/operating system. - -IMDbPY was tested with libxml2 2.7.1, libxslt 1.1.24 and -python-lxml python-lxml 2.1.1. Older versions can work, too; if -you have problems, submit a bug report specifying your versions. - -You can also get the latest version of BeautifulSoup from here: - http://www.crummy.com/software/BeautifulSoup/ -but since it's distributed with IMDbPY, you don't need it (or -you have to override the '_bsoup.py' file in the imdb/parser/http -directory), and this is probably not a good idea, since newer versions -of BeautifulSoup behave in new and unexpected ways. - - - USING THE OLD PARSERS - ===================== - -The old set of parsers was removed since IMDbYP 4.0. - - - FORCING LXML OR BEAUTIFULSOUP - ============================= - -By default, IMdbPY uses python-lxml, if it's installed. -You can force the use of one given parser passing the 'useModule' -parameter. Valid values are 'lxml' and 'BeautifulSoup'. E.g.: - from imdb import IMDb - ia = IMDb('http', useModule='BeautifulSoup') - ... - -useModule can also be a list/tuple of strings, to specify the -preferred order. - - diff -pruN 5.1-1/docs/README.package 6.6-1/docs/README.package --- 5.1-1/docs/README.package 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.package 1970-01-01 00:00:00.000000000 +0000 @@ -1,569 +0,0 @@ - IMDbPY package - ============== - -Here you can find information useful to use IMDbPY to write your -own scripts or programs. -These information are far from complete: the code is the final -documentation! ;-) - -Sections in this file: -* GENERAL USAGE -* THE Movie CLASS -* THE Person CLASS -* THE Character CLASS -* THE Company CLASS -* INFORMATION SETS -* Person OBJECTS INSIDE A Movie CLASS AND Movie OBJECTS INSIDE A Person OBJECT -* Company OBJECTS INSIDE A Movie CLASS AND Movie OBJECTS INSIDE A Company OBJECT -* THE (NOT-SO-)"UNIVERSAL" '::' SEPARATOR -* MOVIE TITLES AND PERSON/CHARACTER NAMES REFERENCES -* EXCEPTIONS -* OTHER SOURCES OF INFO - - UNICODE NOTICE - ============== - -Since release 2.4, IMDbPY internally manages every information about -movies and people using unicode strings. Please read the README.unicode file. - - - GENERAL USAGE - ============= - -To use the IMDbPY package, you've to import the imdb package and -call the IMDb function. -the basic invocation is: - - import imdb - imdb_access = imdb.IMDb() - -If you're accessing a sql installation of the IMDb's data, -you must do: - imdb_access = imdb.IMDb('sql', uri='URI_TO_YOUR_DB') -where 'URI_TO_YOUR_DB' points to your SQL database (see README.sqldb -for more information). - -Now you've the "imdb_access" object, instance of a subclass -of the imdb.IMDbBase class, which can be used to search for a given -title/name and to retrieve information about the referred movie, -person or character. - -The IMDb function can be called with a 'accessSystem' keyword argument, -that must be a string representing the type of data access you want -to use. That's because different systems to access the IMDb data are -available: you can directly fetch data from the web server, you can -have a local copy of the database (see http://www.imdb.com/interfaces/), -you can access movie data through the e-mail interface, etc. etc. - - - Supported access systems | Aliases | Description - ---------------------------+-----------+------------------------------------ - (default) 'http' | 'web', | information are fetched through - | 'html' | the http://akas.imdb.com web server. - ---------------------------+-----------+------------------------------------ - 'sql' | 'db', | information are fetched through - | 'database'| a SQL database (every database - | | supported by SQLObject and SQLAlchemy - | | is available). - ---------------------------+-----------+------------------------------------ - 'mobile' | | same as 'httpThin', but string - | | methods are used for parsing. - | | Ideal for small systems like PDA, - | | smart phones, hand-held devices with - | | limited bandwidth and CPU power. - ---------------------------+-----------+------------------------------------ - 'httpThin' | 'webThin' | identical to 'http', but less - | 'htmlThin'| information are gathered; useful - | | for systems with limited bandwidth. - -NOTE ON THE 'DEFAULT' ACCESS SYSTEM: since release 3.4, the 'imdbpy.cfg' -configuration file is available, so that you can set a system-wide (or -user-wide) default. The file is commented with indication of the location -where it can be put, and how to modify it. -Obviously, if no imdbpy.cfg file is found (or is not readable or it can't -be parsed), 'http' is still considered the default. - - -The imdb_access object has ten main methods: search_movie(title), -get_movie(movieID), search_person(name), get_person(personID), -search_character(name), get_character(characterID), search_company(name), -get_company(companyID), search_episode() and update(MovieOrPersonObject) - - Methods description: - -search_movie(title) searches for the given title, and returns a -list of Movie objects containing only basic information like the -movie title and year, and with a "movieID" instance variable: - - movieID is an identifier of some kind; for the sake of simplicity - you can think of it as the ID used by the IMDb's web server used to - univocally identify a movie (e.g.: '0094226' for Brian De Palma's - "The Untouchables"), but keep in mind that it's not necessary the - same ID!!! - For some implementations of the "data access system" these two IDs can - be the same (and this is the case of the 'http' data access system), but - other "access systems" can use a totally different kind of movieID. - The easier (I hope!) way to understand this is to think at the - movieID of a Movie returned by the search_movie() method as the _thing_ - you've to pass to the get_movie() method, so that it can retrieve info - about the referred movie. - So, movieID _can_ be the imdbID ('0094226') if you're accessing - the web server, but with a sql installation of the IMDb database, - movieID will be an integer, as read from the id columns in the database. - -search_episode(title) is identical to search_movie(), except that its -tailored to search for episodes' titles; best results are expected -searching for just the title of the episode, _without_ the title of -the TV series. - -get_movie(movieID) will fetch the needed data and return a Movie object -for the movie referenced by the given movieID; the Movie class can be -found in the Movie module; a Movie object presents basically the same -interface of a Python's dictionary, so you can access, for example, the -list of actors and actress using the syntax: movieObject['cast'] - -The search_person(name), get_person(personID) search_character(name) -get_character(characterID), search_company(name) and get_company(companyID) -methods work the same way as search_movie(title) and get_movie(movieID). - -The search_keyword(string) method returns a list of unicode string that are -valid keywords, similar to the one given. -The get_keyword(keyword) method returns a list of Movie instances that -are tagged with the given keyword. -For more information see README.keywords. - -The get_imdbMovieID(movieID), get_imdbPersonID(personID), -get_imdbCharacterID(characterID) and get_imdbCompanyID(companyID) take, -respectively, a movieID, a personID, a movieID and a companyID and return -the relative imdbID; it's safer to use the -get_imdbID(MovieOrPersonOrCharacterOrCompanyObject) method. - -The title2imdbID(title), name2imdbID(name), character2imdbID(name) and -company2imdbID(name) take, respectively, a movie title (in the plain text -data files format), a person name, a character name and a company name, and -return the relative imdbID; when possibile it's safer to use the -get_imdbID(MovieOrPersonOrCharacterOrCompanyObject) method. -These functions _always_ need to connect to the IMDb's web site, if -you're not using 'http', 'httpThin' or 'mobile' data acess systems. - -The get_imdbID(MovieOrPersonOrCharacterOrCompanyObject) method returns the -imdbID for the given Movie, Person, Character or Company object. - -The get_imdbURL(MovieOrPersonOrCharacterOrCompanyObject) method returns a -string with the main IMDb URL for the given Movie, Person, Character or -Company object; it tries to do its best to retrieve the URL. - -The update(MovieOrPersonOrCharacterOrCompanyObject) method takes an -instance of a Movie, Person, Character or Company class and retrieve -other available information. -Remember that the search_*(txt) methods will return a list of Movie, Person, -Character or Company objects with only basic information, like the movie -title or the person/character name, so update() can be used to retrieve -every other information. -By default a "reasonable" set of information are retrieved ('main', -'filmography' and 'biography' for a Person/Character objects, 'main' -and 'plot' for a Movie object and 'main' for Company objects). - -Example: - i = IMDb() - # movie_list is a list of Movie objects, with only attributes like 'title' - # and 'year' defined. - movie_list = i.search_movie('the passion') - # the first movie in the list. - first_match = movie_list[0] - # only basic information like the title will be printed. - print first_match.summary() - # update the information for this movie. - i.update(first_match) - # a lot of information will be printed! - print first_match.summary() - # retrieve trivia information and print it. - i.update(first_match, 'trivia') - print m['trivia'] - # retrieve both 'quotes' and 'goofs' information (with a list or tuple) - i.update(m, ['quotes', 'goofs']) - print m['quotes'] - print m['goofs'] - # retrieve every available information. - i.update(m, 'all') - - - THE Movie CLASS - =============== - -The main use of a Movie object is to access to the info it contains -with a dictionary-like interface, like "movieObject[key]" where 'key' -is a string that identifies the information you want to get. - -I've a really bad news for you: at this time, what 'key' is, is a -little unclear! - -In general, it's the name of the section as used by the IMDb web -server to show the data. -Where the information is a list of people with a role (an actor, -a stunt, a writer, etc.) the relative section in the HTML page -starts with a link to a "/Glossary/X#SectName" page; here "sectname" -is used as 'key'. -When the info regard companies (distributors, special effects, etc.) -or the movie itself (sound mix, certifications, etc.) the section -in the HTML page begins with a link to a "/List?SectName=" page, so -we use "sectname" as a 'key'. -The section name (the key) is always (with some minor exceptions) -lowercase; underscores and minus signs are replaced with spaces. -Some other keys aren't taken from the HTML page, but are defined -within the Movie class. -To get the complete list of keys available for a given Movie object, -you can use the movieObject.keys() method (obviously only keys that -refer to some existing information are defined, so a movie without an -art director will raise a KeyError exception is you try -movieObject['art director']); to avoid the exception, you can test -if a Movie object has a given key with the has_key(key) method, or -get the value with the get(key) method, which returns the value or -None if the key is not found (an optional paramenter can modify the -default value returned if the key isn't found). - -Below, a list of the main keys you can encounter, the type of the value -returned by movieObject[key] and a short description/example: - -title; string; the "usual" title of the movie, like "The Untouchables". -long imdb title; string; "Uncommon Valor (1983/II) (TV)" -canonical title; string; the title in the canonical format, - like "Untouchables, The". -long imdb canonical title; string; "Patriot, The (2000)". -year; string; the year of release or '????' if unknown. -kind; string; one in ('movie', 'tv series', 'tv mini series', 'video game', - 'video movie', 'tv movie', 'episode') -imdbIndex; string; the roman number for movies with the same title/year. -director; Person list; a list of director's name (e.g.: ['Brian De Palma']) -cast; Person list; list of actor/actress, with the currentRole instance - variable set to a Character object which describe his - role/duty. -cover url; string; the link to the image of the poster. -writer; Person list; list of writers ['Oscar Fraley (novel)'] -plot; list; list of plots and authors of the plot. -rating; string; user rating on IMDb from 1 to 10 (e.g. '7.8') -votes; string; number of votes (e.g. '24,101') -runtimes; string list; in minutes ['119'] or something like ['USA:118', - 'UK:116'] -number of episodes; int; number or episodes for a series. -color info; string list; ["Color (Technicolor)"] -countries; string list; production's country ['USA', 'Italy'] -genres; string list; one or more in (Action, Adventure, Adult, Animation, - Comedy, Crime, Documentary, Drama, Family, Fantasy, Film-Noir, - Horror, Musical, Mystery, Romance, Sci-Fi, Short, Thriller, - War, Western) and other genres defined by IMDb. -akas; string list; list of aka for this movie -languages; string list; list of languages -certificates; string list; ['UK:15', 'USA:R'] -mpaa; string; the mpaa rating -episodes (series only); dictionary of dictionary; one key for every season, - one key for every episode in the season. -number of episodes (series only); int; total number of episodes. -number of seasons (series only); int; total number of seasons. -series years (series only); string; range of years when the series was produced. -episode of (episode only); Movie object; the parent series for an episode. -season (episode only); int; the season number. -episode (episode only); int; the number of the episode in the season. -long imdb episode title (episode only); string; episode and series title. -series title; string. -canonical series title; string. - - -Other keys that contain a list of Person objects are: costume designer, -sound crew, crewmembers, editor, production manager, visual effects, -assistant director, art department, composer, art director, -cinematographer, make up, stunt performer, producer, set decorator, -production designer. - -Other keys that contain list of companies are: production companies, special -effects, sound mix, special effects companies, miscellaneous companies, -distributors. - -Converting a title to its 'Title, The' canonical format, IMDbPY does -some assumptions about what is an article and what not, and this could -lead to some wrong canonical titles. -For more information on this subject, see the "ARTICLES IN TITLES" -section of the README.locale file. - - - THE Person CLASS - ================ - -It works mostly like the Movie class. :-) - -The Movie class defines a __contains__() method, which is used to -check if a given person has worked in a given movie with the syntax: - if personObject in movieObject: - print '%s worked in %s' % (personObject['name'], movieObject['title']) - -The Person class defines a isSamePerson(otherPersonObject) method, useful -to compare two person if you're not sure that both objects have retrieved -complete information (e.g.: a Person object returned by a query); -th syntax is: - if personObject.isSamePerson(otherPersonObject): - print 'they are the same person!' - -An analogous method is defined for the Movie class, and it's -called isSameTitle(otherMovieObject) - - - THE Character CLASS - =================== - -It works mostly like the Person class. :-) -For more information about the "currentRole" attribute, see the -README.currentRole file. - - - THE Company CLASS - ================ - -It works mostly like the Person class. :-) -The "currentRole" attribute is always None. - - - INFORMATION SETS - ================ - -Since release 1.2, it's possibile to retrieve almost every piece of -information about a given movie or person; this can be a problem, because -(at least for the 'http' data access system) it means that a lot of -web pages must be fetched and parsed, and this can be time and -bandwidth consuming, especially if you're interested only in a small set -of information. - -Now the get_person, get_movie, get_character, get_company and update -methods have an optional 'info' argument, which can be set to a list -of strings, each one representing an "information set". -Movie/Person/Character/Company objects have, respectively, their own list -of available "information sets". -E.g.: the Movie class have a set called 'taglines' for the taglines -of the movie, a set called 'vote details' for the number of votes for -rating [1-10], demographic breakdowns and top 250 rank; the Person -class have a set called 'other works' for miscellaneous works of -this person an so on. - -By default only important information are retrieved/updated (i.e.: -for a Movie object, only the 'main' and 'plot' information sets; -for a Person/Character object only 'main', 'filmography', 'biography'. - -Example: - i = imdb.IMDb(accessSystem='http') - m = i.get_movie('0133093') # only default info set are retrieved. - m.has_key('demographic') # returns false, since no demographic breakdowns - # aren't available by default. - i.update(m, info=('vote details',)) # retrieve the vote details info set. - print m['demographic'] # print the demographic breakdowns. - -Another example: - i = imdb.IMDb(accessSystem='http') - # retrieve only the biography and the "other works" page: - p = i.get_person('0000154', info=['biography', 'other works']) - print p['salary'] - print p['other works'] - -To see which information sets are available and what are the defaults, -see the all_info and default_info instance variable of Movie, Person -and Character classes. Each object instance of Movie, Person or Character, -also have a current_info instance variable, to remember the information sets -already retrieved. - -Beware that the information sets vary from an access system to another: -locally not every data is accessible, while - for example for sql - -accessing one set of data automatically means automatic access to a number -of other unrelated information (without major performace drawbacks). -You can get the list of available info set with the methods: -i.get_movie_infoset(), i.get_person_infoset(), i.get_character_infoset() -and i.get_company_infoset(). - - - TOP250 / BOTTOM100 LISTS - ======================== - -Since IMDbPY 4.0, it's possible to retrieve the list of top250 -and bottom100 movies. -Use the get_top250_movies() and get_bottom100_movies() methods. -Beware that, for 'sql', the bottom100 list i limited to the first 10 results. - - - Person OBJECTS INSIDE A Movie CLASS AND Movie OBJECTS INSIDE A Person OBJECT - ============================================================================ - -Parsing the information about a movie, you'll encounter a lot of references -to the people who worked on it, like the cast, the director, the stunts, -and so on. -For people in the cast (actors/actresses), the "currentRole" instance -variable is set to the name of the character they played (e.g.: "Roy Neary" -for the role played by Richard Dreyfuss in Close Encounters of the Third Kind). -In fact, in this case currentRole will be a Character instance. - -Another instance variable of a Person object is "notes", used to store -miscellaneous information (like an aka name for the actor, an "uncredited" -notice and so on). -It's also used, for non-cast people, to describe the specific task of -the person (e.g.: "assistant dialogue staff" for a person of the sound -departement). - -It's possible to test, with the Python "in" statement, if a person worked -in a given movie, or vice-versa; the following are all valid tests: - movie in person - movie in character - person in movie - person in character - character in movie - character in person - -Considerations similar to the above ones, can be done for Character -instances: please read the README.currentRole file for more information. - -E.g.: - # retrieve data for Steven Spielberg's "Close Encounters of the Third Kind" - import imdb - i = imdb.IMDb(accessSystem='http') - movie = i.get_movie('0075860') - - # Get the 7th Person object in the cast list - cast = movie['cast'][6] - # Will print "Warren J. Kemmerling" - print cast['name'] - # Will print "Wild Bill" - print cast.currentRole - # Will print "(as Warren Kemmerling)" - print cast.notes - - # Get the 5th Person object in the list of writers - writer = movie['writer'][4] - # Will print "Steven Spielberg" - print writer['name'] - # Will print "written by", because that was duty of Steven Spielberg, - # as a writer for the movie. - print writer.notes - -Obviously these Person objects contain only information directly -available parsing the movie pages (e.g.: the name, an imdbID, the role/duty), -so if now you: - print writer['actor'] -to get a list of movies acted by Mel Gibson, you'll get a KeyError -exception, because the Person object doesn't contain this kind of -information. - -To gather every available information, you've to use the update() -method of the IMDb class: - i.update(writer) - # Now it will print a list of Movie objects. - print writer['actor'] - -The same is true parsing a person data: you'll find a list of movie -he/she worked on and, for every movie, the currentRole instance variable -is set to a string describing the role/duty of the considered person. -E.g.: - # Julia Roberts - julia = i.get_person('0000210') - # Print a list of movies she acted in and the played role, separated - # by '::' - print [movie['title'] + '::' + movie.currentRole - for movie in julia['actress']] - -Here the various Movie objects only contain minimal information, like -the title and the year; the latest movie with Julia Roberts: - last = julia['actress'][0] - # Retrieve full information - i.update(last) - # Print the name of the first director - print last['director'][0]['name'] - - - Company OBJECTS INSIDE A Movie CLASS AND Movie OBJECTS INSIDE A Company OBJECT - ============================================================================== - -As for Person/Character and Movie objects, you can test - using the "in" -operator - if a Company has worked on a given Movie. - - - THE (NOT-SO-)"UNIVERSAL" '::' SEPARATOR - ======================================= - -Sometimes I've used '::' to separate a set of different information -inside a string, like the name of a company and what it has done for the -movie, the information in the "Also Known As" section, and so on. -It's easier to understand if you look at it; look at the output of: - import imdb - i = imdb.IMDb() - m = i.get_movie('0094226') - print m['akas'] - -As a rule, there's as much as one '::' separator inside a string, -splitting it two logical pieces: "TEXT::NOTE". -In the helpers module there's the makeTextNotes function, that can -be used to create a custom function to pretty-print this kind of -information. See its documentation for more info. - - - MOVIE TITLES AND PERSON/CHARACTER NAMES REFERENCES - ================================================== - -Sometimes in Movie, Person and Character attributes, there're strings -with references to other movies or persons (e.g.: in the plot, in -the biography, etc.). -These references are stored in the Movie, Person and Character -instances; in the strings you'll find values like _A Movie (2003)_ (qv) -or 'A Person' (qv) or '#A Character# (qv)'; accessing these -string (like movie['plot'] or person['biography']), these strings are -modified using a provided function, which must take, as arguments, the -string and two dictionary with titles and names references; -by default the (qv) strings are converted in the "normal" -format ("A Movie (2003)", "A Person" and "A Character"). -You can find some examples of these functions in the -imdb.utils module. -The function used to modify the strings can be set with -the defaultModFunct parameter of the IMDb class or -with the modFunct parameter of the get_movie, get_person -and get_character methods. -E.g.: - import imdb - i = imdb.IMDb(defaultModFunct=imdb.utils.modHtmlLinks) - -Or: - import imdb - i = imdb.IMDb() - i.get_person('0000154', modFunct=imdb.utils.modHtmlLinks) - - - EXCEPTIONS - ========== - -The imdb._exceptions module contains the exceptions raised by the -imdb package. Every exception is a subsclass of IMDbError, which is -available from the imdb package. - -You can catch any type of errors raised by the IMDbPY package with -something like: - from imdb import IMDb, IMDbError - - try: - i = IMDb() - except IMDbError, err: - print err - - try: - results = i.search_person('Mel Gibson') - except IMDbError, err: - print err - - try: - movie = i.get_movie('0335345') - except IMDbError, err: - print err - - - OTHER SOURCES OF INFO - ===================== - -Once the IMDbPY package is installed, you can read the docstring for -packages, modules, functions, classes, objects, methods using the -pydoc program; e.g.: "pydoc imdb.IMDb" will show the documentation -about the imdb.IMDb class. - -The code contains a lot of comments, try reading it, if you can -understand my English! - - diff -pruN 5.1-1/docs/README.redesign 6.6-1/docs/README.redesign --- 5.1-1/docs/README.redesign 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.redesign 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ - - IMDb's web site redesign - ======================== - -On September 2010 the IMDb web pages had a major redesign. -With IMDbPY 4.7 we're trying to parse the new web pages, -but it will take some time before all the bugs are fixed. - -Any help (fixing parsers or simple bug reports) is greatly -appreciated. - -Beware that: -- the "httpThin" data access method is badly broken and - probably it will not fixed. -- the "mobile" data access method can be partially broken, - and will be fixed: please report any problem. -- some of the information in these keys could be somewhat - ruined: soundtrack, awards, episodes rating, faqs. -- information about series were not extensively tested. -- it's possible that some information will be missing, like - "in development" movies. - -The above problems, with the exception of "httpThin" will be -fixed in future releases. - -Notes about the code: we have a very powerful and efficient -parsing infrastructure, but after many releases and so many -changes of the IMDb pages, some of the main parsers are showing -their age. So, parsers for main information about movies and -persons should be probably rewritten from scratch, and the same -applies to helper functions like "build_person" and "build_movie" -in imdb.parser.http.utils. - diff -pruN 5.1-1/docs/README.series 6.6-1/docs/README.series --- 5.1-1/docs/README.series 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.series 1970-01-01 00:00:00.000000000 +0000 @@ -1,205 +0,0 @@ -Summary of this file: -* MANAGING SERIES EPISODES -* TITLES -* SERIES -* FULL CREDITS -* RATINGS -* PEOPLE -* GOODIES - - - MANAGING SERIES EPISODES - ======================== - -Since January 2006, IMDb changed the way it handles TV episodes: -now every episode is treated as full title. -Starting with version 2.5, also IMDbPY supports this new behavior. - - - TITLES - ====== - -analyze_title() and build_title() now supports tv episodes. -You can pass a string to the analyze_title function in the format used -by the web server ("The Series" The Episode (2005)) or in the format -of the plain text data files ("The Series" (2004) {The Episode (#ser.epi)}) - -An example of the returned dictionary: call the function: - analyze_title('"The Series" The Episode (2005)') - -the result will be: - {'kind': 'episode', # kind is set to 'episode'. - 'year': '2005', # the release year of this episode. - 'title': 'The Episode', # episode title - 'episode of': {'kind': 'tv series', # 'episode of' will contains - 'title': 'The Series'} # information about the series. - } - - -The 'episode of' key can be a dictionary or a Movie class instance with -the same information. - -The build_title() function takes an optional argument: ptdf; is it's -set to false (the default), it returns the title of the episode in -the format used by the IMDb's web server ("The Series" An Episode (2006)), -otherwise it uses the format used by the plain text data files (something -like "The Series" (2004) {An Episode (#2.5)}) - - - SERIES - ====== - -You can retrieve information about seasons and episodes for a tv (mini) series: - - from imdb import IMDb - i = IMDb() - m = i.get_movie('0389564') # The 4400. - m['kind'] # kind is 'tv series'. - i.update(m, 'episodes') # retrieves episodes information. - - m['episodes'] # a dictionary with the format: - # {#season_number: { - # #episode_number: Movie object, - # #episode_number: Movie object, - # ... - # }, - # ... - # } - # season_number always starts with 1, episode_number - # depends on the series' numbering schema: some series - # have a 'episode 0', while others starts counting from 1. - - m['episodes'][1][1] # - - e = m['episodes'][1][2] # second episode of the first season. - e['kind'] # kind is 'episode'. - e['season'], e['episode'] # return 1, 2. - e['episode of'] # - # XXX: beware that e['episode of'] and m _are not_ the - # same object, while both represents the same series. - # This is to avoid circular references; the - # e['episode of'] object only contains basics - # information (title, movieID, year, ....) - i.update(e) # retrieve normal information about this episode (cast, ...) - - e['title'] # 'The New and Improved Carl Morrissey' - e['series title'] # 'The 4400' - e['long imdb episode title'] # '"The 4400" The New and Improved Carl Morrissey (2004)' - - -Summary of keys of the Movie object for a series episode: - 'kind': set to 'episode'. - 'episode of': set to a movie object, this is a reference to the series. - 'season': an integer; the number of the season. - 'episode': an integer; the number of the episode in the season. - 'long imdb episode title': combines series and episode title. - 'series title': title of the series. - 'canonical series title': title of the series, in the canonical format. - -Summary of keys of the Movie object for a series: - 'kind': set to 'tv series'. - 'episodes': dictionary (seasons) of dictionary (episodes in the season). - - - FULL CREDITS - ============ - -Retrieving credits for a tv (mini) series, you may notice that many long lists -(like "cast", "writers", ...) are incomplete. -You can fetch the complete list of cast and crew with the "full credits" -data set; e.g.: - from imdb import IMDb - i = IMDb() - m = i.get_movie('0285331') # 24. - print len(m['cast']) # wooah! Only 7 person in the cast of 24?!?! - i.update(m, 'full credits') - print len(m['cast']) # yup! More than 300 persons! - -If you prefer, you can retrieve the complete cast of every episode, -keeping the lists separated for every episode; instead of retrieving -the list of episodes with: - i.update(m, 'episodes') -use instead: - i.update('episodes cast') -or the equivalent: - i.update(m, 'guests') - -Now you end up having the same information as if you have updated -the 'episodes' info set, but every Movie object inside the dictionary -of dictionary has the complete cast. -E.g.: - cast = m['episodes'][1][2]['cast'] # cast list for the second episode - # of the first season. - -Beware that both 'episodes cast' and 'guests' will update the -keyword 'episodes' (and not 'episodes cast' or 'guests'). - - - RATINGS - ======= - -You can retrieve rating information about every episode in a tv (mini) series -using the 'episodes rating' data set. - - - PEOPLE - ====== - -You can retrieve information about single episodes acted/directed/... by -a person. - - from imdb import IMDb - i = IMDb() - p = i.get_person('0005041') # Laura Innes. - p['actress'][0] # - - # At this point you have an entry (in keys like 'actor', 'actress', - # 'director', ...) for every series the person starred/worked in, but - # you knows nothing about singles episodes. - i.update(p, 'episodes') # updates information about single episodes. - - p['episodes'] # a dictionary with the format: - # {: [ - , - , - ... - ], - ... - } - - er = p['actress'][0] # ER tv series. - p['episodes'][er] # list of Movie objects; one for every ER episode - # she starred/worked in. - - p['episodes'][er][0] # - p['episodes'][er]['kind'] # 'episode' - p['episodes'][er][0].currentRole # 'Dr. Kerry Weaver' - - - GOODIES - ======= - -In the imdb.helpers module there are some functions useful to manage -lists of episodes: - -- sortedSeasons(m) returns a sorted list of seasons of the given series. - E.g.: - >>> from imdb import IMDb - >>> i = IMDb() - >>> m = i.get_movie('0411008') - >>> i.update(m, 'episodes') - >>> sortedSeasons(m) - [1, 2] - -- sortedEpisodes(m, season=None) returns a sorted list of episodes of the - the given series, considering only the - specified season(s) (every season, if None). - E.g.: - >>> from imdb import IMDb - >>> i = IMDb() - >>> m = i.get_movie('0411008') - >>> i.update(m, 'episodes') - >>> sortedEpisodes(m, season=1) - [, , ...] - - diff -pruN 5.1-1/docs/README.sqldb 6.6-1/docs/README.sqldb --- 5.1-1/docs/README.sqldb 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.sqldb 1970-01-01 00:00:00.000000000 +0000 @@ -1,399 +0,0 @@ - -NOTE: the imdbpy2sql.py script, used to populate a database using -the data in the IMDb's plain text data files, is a critical piece -of IMDbPY: it's based on an ORM to be database-independent and -contains a lot of tricks to be as fast as possible; however there -are huge margins for improvements; if you want to help, please read the -TODO.txt file and subscribe the imdbpy-devel mailing list at: - http://imdbpy.sf.net/?page=help#ml - - -NOTE: see README.currentRole for information about characters support. - - - SQL - === - -Since version 2.1 it's possible to transfer the whole IMDb's -database from the plain text data files into a SQL database. -Starting with version 2.5 every database supported by the SQLObject -Object Relational Manager can be used to store and retrieve -movies and persons information. -This means that MySQL, PostgreSQL, SQLite, Firebird, MAX DB, -Sybase and MSSQL are supported and, as your read this text, -maybe other database backends were added. - -Since release 3.8, SQLAlchemy (version 0.4 and 0.5) is also supported -(this adds at least DB2/Informix IDS to the list of supported databases). - -Since release 3.9, there's a partial support to output large tables -in a set of CSV (Comma Separated Values) files, to be later imported -in a database. Actually only MySQL, PostgreSQL and IBM DB2 are -supported. - -In version 4.1 the imdbpy2sql.py script has the '--fix-old-style-titles' -command line argument; if used, every movie title will be converted to -the new style ("The Title", instead of the old "Title, The"). -This option will go away in 4.2, and is intended only to support old -set of plain text data files. - -Since version 4.2 --fix-old-style-titles is no more needed, being -turned on by default. The --do-not-fix-old-style-titles was -introduced in case you want to turn it off for some strange reason. - - - REQUIREMENTS - ============ - -You need one of SQLObject or SQLAlchemy (both can be installed -safely: by default IMDbPY first tries SQLObject; if not present -it fall-backs to SQLAlchemy). - -[SQLObject] -You need the SQLObject package, at least version 0.8; even better -if you can download the latest SVN snapshot. - -SQLObject home page: http://sqlobject.org/ -SVN command to download the latest development version: - svn co http://svn.colorstudy.com/SQLObject/trunk SQLObject - -[SQLAlchemy] -Support for SQLAlchemy is still in beta (please report any bug!) -and a bit slower than SQLObject; anyway, you need version 0.4 or 0.5. - -SQLAlchemy home page: http://www.sqlalchemy.org/ -SVN command to download the latest development version: - svn checkout http://svn.sqlalchemy.org/sqlalchemy/trunk sqlalchemy - - -[OTHER REQUIRED MODULES] -Obviously SQLObject and SQLAlchemy can access databases only through other -specific modules/packages, that you need to have installed (e.g.: -'mysql-python' for MySQL, 'psycopg' for PostgreSQL, and so on). - - - SQL DATABASE INSTALLATION - ========================= - -Select a mirror of the "The Plain Text Data Files" from -the http://www.imdb.com/interfaces/ page and download -every file in the main directory (beware that the "diffs" -subdirectory contains _a lot_ of files you _don't_ need, -so don't start mirroring everything!). - -Starting from release 2.4, you can just download the files you need, -instead of every single file; the files not downloaded will be skipped. -This feature is still quite untested, so please report any bug. - -Create a database named "imdb" (or whatever you like), -using the tool provided by your database; as an example, for MySQL -you will use the 'mysqladmin' command: - # mysqladmin -p create imdb -For PostgreSQL, you have to use the "createdb" command: - # createdb -W imdb - -To create the tables and to populate the database, you must run -the imdbpy2sql.py script: - # imdbpy2sql.py -d /dir/with/plainTextDataFiles/ -u 'URI' - -Where the 'URI' argument is a string representing the connection -to your database, with the schema: - scheme://[user[:password]@]host[:port]/database[?parameters] - -Where 'scheme' is one in "sqlite", "mysql", "postgres", "firebird", -"interbase", "maxdb", "sapdb", "mssql", "sybase", "ibm_db_sa". - -Some examples: - mysql://user:password@host/database - postgres://user:password@host/database - mysql://host/database?debug=1 - postgres:///full/path/to/socket/database - postgres://host:5432/database - sqlite:///full/path/to/database - sqlite:/C|/full/path/to/database - sqlite:/:memory: - -For other information you can read the SQLObject/SQLAlchemy documentation. -You can force the use of SQLObject or SQLAlchemy with the '-o' command -line option (i.e.: "-o sqlobject" or "-o sqlalchemy" or a list of comma -separated values to specify an order of preference). - - - TIMING - ====== - -The performances are hugely dependant upon the underlying Python -module/package used to access the database. The imdbpy2sql.py script -has a number of command line arguments, useful to chose amongst -presets that can improve performances, using specific database servers. - -The fastest database appears to be MySQL, with about 200 minutes to -complete on my test system (read below). -A lot of memory (RAM or swap space) is required, in the range of -at least 250/500 megabytes (plus more for the database server). -In the end, the database will require between 2.5GB and 5GB of disc space. -The should be no difference - at insert time - between SQLObject and -SQLAlchemy. - -As said, the performances varies greatly using a database server or another: -MySQL, for instance, has an executemany() method of the cursor object -that accept multiple data insertion with a single SQL statement; other -database requires a call to the execute() method for every single row -of data, and they will be much slower - from 2 to 7 times slower than -MySQL. - -There are generic suggestions that can lead to better performances, -like turning off your filesystem journaling (so it can be a good idea to -remount an ext3 filesystem as ext2). Another option is the use of a -ramdisk/tmpfs, if you have enough RAM. Obviously these have effect only -at insert-time: during the day-to-day use, you can turn your journaling -on again. You can also consider the use of the CSV output, explained -below (but be sure that your database server of choice is able to -import CSV files). - -I've done some tests, using an AMD Athlon 1800+, 1GB of RAM, over a -complete plain text data files set (as of 11 Apr 2008, with more than -1.200.000 titles and over 2.200.000 names): - - database | time in minutes: total (insert data/create indexes) - ----------------------+----------------------------------------------------- - MySQL 5.0 MyISAM | 205 (160/45) - MySQL 5.0 InnoDB | _untested_, see NOTES below. - PostgreSQL 8.1 | 560 (530/30) - SQLite 3.3 | ??? (150/???) - very slow building indexes. - | Timed with the "--sqlite-transactions" command - | line option; otherwise it's _really_ slow: even - | 35 hours or more. - SQLite 3.7 | 65/13 - with --sqlite-transactions and using a SSD hard disk - SQL Server | about 3 or 4 hours. - -If you have different experiences, please tell me! -As expected, the most important things that you can do to improve performances are: -1. use an in-memory filesystem or an SSD disk. -2. use the -c /path/to/empty/dir argument to use CSV files. -3. follow the specific notes about your database server. - - - NOTES - ===== - -[save the output] -The imdbpy2sql.py will print a lot of debug information on standard output; -you can save it in a file, appending (without quotes) "2>&1 | tee output.txt" - - -[Microsoft Windows paths] -It's much safer, in a Microsoft Windows environment, to use full paths -for the values of the '-c' and '-d' arguments, complete with drive letter. -The best thing is to use _UNIX_ path separator, and to add a leading -separator. E.g.: - -d C:/path/to/imdb_files/ -c C:/path/to/csv_tmp_files/ - - -[MySQL] -In general, if you get an embarrassingly high numbero of "TOO MANY DATA -... SPLITTING" lines, consider increasing max_allowed_packet (in the -configuration of your MySQL server) to at least 8M or 16M. -Otherwise, inserting the data will be very slow, and some data may -be lost. - - -[MySQL InnoDB and MyISAM] -InnoDB is abysmal slow for our purposes: my suggestion is to always -use MyISAM tables and - if you really want to use InnoDB - convert -the tables later. -The imdbpy2sql.py script provides a simple way to manage these cases, -see ADVANCED FEATURES below. - -In my opinion, the cleaner thing to do is to set the server to use -MyISAM tables or - you you can't modifiy the server - use the ---mysql-force-myisam command line option of imdbpy2sql.py. -Anyway, if you really need to use InnoDB, in the server-side settings -I recommend to set innodb_file_per_table to "true". - -Beware that the conversion will be extremely slow (some hours), but -still faster than using InnoDB from the begin. -You can use the "--mysql-innodb" command line option to force the -creation of a datbase with MyISAM tables, converted at the end -into InnoDB. - - -[Microsoft SQL Server/SQLExpress] -If you get and error about how wrong and against nature is the -blasphemous act of inserting indentity keys, you can try to fix it -with the new custom queries support; see ADVANCED FEATURES below. - -As a shortcut, you can use the "--ms-sqlserver" command line option -to set all the needed options. -You probably need SQLObject 0.10 (in the svn repository, as I'm -writing this). - - -[SQLite speed-up] -For some reason, SQLite is really slow, except when used with -transactions; you can use the '--sqlite-transactions' command -line option to obtain acceptable performances. -The same command, also turns off "PRAGMA synchronous". - -SQLite seems to hugely benefit from the use of a non-journaling -filesystem and/or of a ramdisk/tmpfs: see the generic suggestions -discussed above in the TIMING section. - - -[SQLite failure] -It seems that, with older versions of the python-sqlite package, the first -run may fail; if you get a DatabaseError exception saying "no such table", -try running again the command with the same arguments. Double funny, uh? ;-) - - -[data truncated] -If you get an insane amount (hundreds or thousands, on various text -columns) of warnings like these lines: - - imdbpy2sql.py:727: Warning: Data truncated for column 'person_role' at row 4979 - CURS.executemany(self.sqlString, self.converter(self.values())) - -you probably have a problem with the configuration of your database. -The error came from strings that get cut at the first non-ASCII char (and -so you're losing a lot of information). -To obviate at this problem, you must be sure that your database -server is set up properly, with the use library/client configured -to communicate with the server in a consistent way. -E.g., for MySQL you can set: - character-set-server = utf8 - default-collation = utf8_unicode_ci - default-character-set = utf8 - -of even: - character-set-server = latin1 - default-collation = latin1_bin - default-character-set = latin1 - - -[adult titles] -Beware that, while running, the imdbpy2sql.py script will output a lot -of strings containing both person names and movie titles. The script -has absolutely no way to know that the processed title is an adult-only -movie, so... if you leave it running and your little daughter runs to you -screaming 'daddy! daddy! what kind of animals Rocco trains in the -documentary "Rocco: Animal Trainer 17"???'... well it's not my fault! ;-) - - - SQL USAGE - ========= - -Now you can use IMDbPY with the database: - from imdb import IMDb - i = IMDb('sql', uri='YOUR_URI_STRING') - resList = i.search_movie('the incredibles') - for x in resList: print x - ti = resList[0] - i.update(ti) - print ti['director'][0] - -and so on... - -The 'sql' data access system takes an optional argument, named "useORM", -which can be set to a string or a list of values (the string can be -a comma-separated list of items, to denote an order of preference). -Valid values are "sqlobject" and "sqlalchemy". -The default is ('sqlobject', 'sqlalchemy'). -E.g.: - i = IMDb('sql', uri='YOUR_URI_STRING', useORM='sqlalchemy,sqlobject') - i = IMDb('sql', uri='YOUR_URI_STRING', useORM=['sqlalchemy', 'sqlobject']) - i = IMDb('sql', uri='YOUR_URI_STRING', useORM='sqlalchemy']) - - - ADVANCED FEATURES - ================= - -With the -e (or --execute) command line argument you can specify -custom queries to be executed at certain times, with the syntax: - -e "TIME:[OPTIONAL_MODIFIER:]QUERY" - -Where TIME is actually one of these: 'BEGIN', 'BEFORE_DROP', 'BEFORE_CREATE', -'AFTER_CREATE', 'BEFORE_MOVIES', 'BEFORE_CAST', 'BEFORE_RESTORE', -'BEFORE_INDEXES' and 'END'. - -The only available OPTIONAL_MODIFIER is 'FOR_EVERY_TABLE' and it -means that the QUERY command will be executed for every table in the -database (so it doesn't make much sense to use it with BEGIN, BEFORE_DROP -or BEFORE_CREATE time...), replacing the "%(table)s" text in the QUERY -with the appropriate table name. - -Other available TIMEs are: 'BEFORE_MOVIES_TODB', 'AFTER_MOVIES_TODB', -'BEFORE_PERSONS_TODB', 'AFTER_PERSONS_TODB', 'BEFORE_CHARACTERS_TODB', -'AFTER_CHARACTERS_TODB', 'BEFORE_SQLDATA_TODB', 'AFTER_SQLDATA_TODB', -'BEFORE_AKAMOVIES_TODB' and 'AFTER_AKAMOVIES_TODB'; they take no modifiers. -Special TIMEs 'BEFORE_EVERY_TODB' and 'AFTER_EVERY_TODB' apply to -every BEFORE_* and AFTER_* TIME above mentioned. -These commands are executed before and after every _toDB() call in -their respective objects (CACHE_MID, CACHE_PID and SQLData instances); -the "%(table)s" text in the QUERY is replaced as above. - -You can specify so many -e arguments as you need, even if they -refers to the same TIME: they will be executed from the first to the last. -Also, always remember to correctly escape queries: after all you're -passing it on the command line! - -E.g. (ok, quite a silly example...): - -e "AFTER_CREATE:SELECT * FROM title;" - -The most useful case is when you want to convert the tables of a MySQL -from MyISAM to InnoDB: - -e "END:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;" - -If your system uses InnoDB by default, you can trick it with: - -e "AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;" -e "END:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;" - -You can use the "--mysql-innodb" command line option as a shortcut -of the above command. - -Cool, uh? - -Another possible use is to fix a problem with Microsoft SQLServer/SQLExpress: -to prevent errors setting IDENTITY fields, you can run something like this: - -e 'BEFORE_EVERY_TODB:SET IDENTITY_INSERT %(table)s ON' -e 'AFTER_EVERY_TODB:SET IDENTITY_INSERT %(table)s OFF' - -You can use the "--ms-sqlserver" command line option as a shortcut -of the above command. - -To use transactions to speed-up SQLite, try: - -e 'BEFORE_EVERY_TODB:BEGIN TRANSACTION;' -e 'AFTER_EVERY_TODB:COMMIT;' - -Which is also the same thing the command line option '--sqlite-transactions' -does. - - - CSV files - ========= - -Keep in mind that actually only MySQL, PostgreSQL and IBM DB2 are -supported. Moreover, you may incur in problems (e.g.: your -postgres _server_ process must have reading access to the directory -you're storing the CSV files). - -To create (and import) a set of CSV files, run imdbpy2sql.py with the -syntax: - ./imdbpy2sql.py -d /dir/with/plainTextDataFiles/ -u URI -c /directory/where/to/store/CSVfiles - -The created files will be imported near the end of the imdbpy2sql.py -processing; notice that after that, you can safely cancel these files. - - - CSV partial processing - ====================== - -It's possible, since IMDbPY 4.5, to separate the two steps involved using -CSV files. -With the --csv-only-write command line option the old database will -be zeroed and the CSV files saved (along with imdbIDs information). -Using the --csv-only-load option you can load these saved files into -an existing database (this database MUST be the one left almost empty -by the previous run). -Beware that right now the whole procedure is not very well tested. -Using both commands, on the command line you still have to specify -the whole "-u URI -d /path/plainTextDataFiles/ -c /path/CSVfiles/" -series of arguments. - - diff -pruN 5.1-1/docs/README.txt 6.6-1/docs/README.txt --- 5.1-1/docs/README.txt 2016-03-28 10:28:18.000000000 +0000 +++ 6.6-1/docs/README.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,215 +0,0 @@ - What's IMDbPY? - ============== - -NOTE: see also the recommendations in the "DISCLAIMER.txt" file. - - -IMDbPY is a Python package useful to retrieve and manage the data of -the IMDb movie database. - -IMDbPY is mainly a tool intended for programmers and developers, but -some example scripts are included. - -If you're a poor, simple, clueless user, read the "README.users" file. :-) -Seriously: take a look at the provided example scripts even if you're -a Really Mighty Programmer(tm), they should clearly show how to use IMDbPY. -Other IMDbPY-based programs can be downloaded from: - http://imdbpy.sourceforge.net/?page=programs - -If you want to develop a program/script/package/framework using the -IMDbPY package, see the "README.package" file, for instructions about -how to use this package. - -If you're installing IMDbPY in a smart phone, PDA or hand-held system, -read the "README.mobile" file. - -If you're crazy enough and/or you've realized that your higher -inspiration in life is to help the development of IMDbPY, begin reading -the "README.devel" file. ;-) - - - INSTALLATION - ============ - -Everything you need to do is to run, as the root user, the command: - # python setup.py install - -IMDbPY itself can be installed through easy_install and pip, -with - respectively - these commands (as root): - easy_install IMDbPY - pip install IMDbPY - -Using easy_install and pip, the dependencies will be automatically -satisfied. Third-party packages may be downloaded, and if not -otherwise specified (see below), C extensions compiled (this means -that you need the python-dev package installed). - -If, for some reason, it doesn't work, you can copy the "./imdb" -directory in the local site-packages directory of the python -major version you're using, but remember that you'll not satisfy -the required dependencies and neither compile the optional C module, -so use this as your very last resort. -To know what major version of python you've installed, run: - $ python -V -It should return a string like "Python 2.6.1"; in this example -the major version is "2.6". -Now copy the "./imdb" directory: - # cp -r ./imdb /usr/local/lib/python{MAJORVERSION}/site-packages/ - -The setup.py contains some configuration options that could -be useful if you're installing IMDbPY in a system with very -little hard disk space (like an handheld device) or where -you've not a complete development environment available; -read the "README.mobile" file. - -If you want to insert the content of the plain text data files -into a SQL database, read the "README.sqldb" file. - -The whole list of command line options of the setup.py script is: - --without-lxml exclude lxml (speeds up "http" considerably, - so try to fix it). - --without-cutils don't compile the C module (speeds up 'sql') - --without-sqlobject exclude SQLObject - --without-sqlalchemy exclude SQLAlchemy - -By default, setup.py tries to install BOTH SQLObject -and SQLAlchemy. In fact, having one of them will be enough: -you can exclude the unwanted one. - - - Mercurial VERSION - ================= - -The best thing is always to use a package for your distribution, -or use easy_install or pip to install the latest release, but it -goes without saying that sometimes you need the very latest version -(keep in mind that the IMDb site is a moving target...). -In this case, you can always use the Mercurial version, available here: - http://imdbpy.sourceforge.net/?page=download#hg - - - HELP - ==== - -Refer to the web site http://imdbpy.sf.net/ and subscribe to the -mailing list: http://imdbpy.sf.net/?page=help#ml - - - NOTES FOR PACKAGERS - =================== - -If you plan to package IMDbPY for your distribution/operating system, -keep in mind that, while IMDbPY can works out-of-the-box, some external -package may be required for certain functionality: - - python-lxml: the 'http' data access system will be much faster, if - it's installed. - - SQLObject or SQLAlchemy: one of these is REQUIRED if you want to use - the 'sql' data access system. - -All of them should probably be "recommended" (or at least "suggested") -dependencies. -To compile the C module, you also need the python-dev package. - -As of IMDbPY 4.0, the installer is based on setuptools. - - - RECENT IMPORTANT CHANGES - ======================== - -Since release 2.4, IMDbPY internally manages every information about -movies and people using unicode strings. Please read the README.unicode file. - -Since release 3.3, IMDbPY supports IMDb's character pages; see the -README.currentRole file for more information. - -Since release 3.6, IMDbPY supports IMDb's company pages; see the -README.companies file for more information. - -Since release 3.7, IMDbPY has moved its main parsers from a SAX-based -approach to a DOM/XPath-based one; see the README.newparsers file -for more information. - -Since release 3.8, IMDbPY supports both SQLObject and SQLAlchemy; see -README.sqldb for more information. - -Since release 3.9 support dumping the plain text data files in CSV files; -see README.sqldb for more information. - -Since release 4.0 it's possible to search for keywords (get keywords -similar to a given one and get a list of movies for a specified keyword). -See README.keywords for more information. -Moreover, it's possible to get information out of Movie, Person, Character -and Company instances as XML (getting a single keys or the representation -of a whole object). -See README.info2xml for more information. -Another new feature, is the ability to get top250 and bottom100 lists; -see the "TOP250 / BOTTOM100 LISTS" section of the README.package file -for more information. - -Since release 4.1 a DTD for the XML output is available (see -imdbpyXY.dtd). Other important features are locale (i18n) support (see -README.locale) and support for the new style of movie titles used by IMDb -(now in the "The Title" style, and no more as "Title, The"). - - - FEATURES - ======== - -So far you can search for a movie with a given title, a person -with a given name, a character you've seen in a movie or a company, and retrieve -information for a given movie, person, character or company; the supported data -access systems are 'http' (i.e.: the data are fetched through the IMDb's -web server http://akas.imdb.com) and 'sql', meaning that the data are -taken from a SQL database, populated (using the imdbpy2sql.py script) with -data taken from the plain text data files; see -http://www.imdb.com/interfaces/ for more information. -For mobile systems there's the 'mobile' data access system, useful -for PDA, hand-held devices and smart phones. -Another data access system is 'httpThin', which is equal to 'http' -but fetch less data and so it is (or at least it tries to be) -suitable for systems with limited bandwidth but normal CPU power. - - - FEATURES OF THE HTTP DATA ACCESS SYSTEM - ======================================= - -* Returns almost every available information about a movie, person or - character. -* The use of the "akas" server will provide access to a lot of - AKA titles in many languages, so it's really useful if English is - not your native language. -* By default includes adult titles (and people who have worked - only/mostly in adult movies) in the results of a title/name search; this - behavior can be changed with the do_adult_search() method; please - read the "README.adult" file. -* You can set/use a proxy to access the web; if set, the HTTP_PROXY - environment variable will be automatically used, otherwise you can set a - proxy with the set_proxy() method of the class returned by the - imdb.IMDb function; obviously this method is available only for the http - data access system, since it's defined in the IMDbHTTPAccessSystem class - of the parser.http package. - Example: - from imdb import IMDb - i = IMDb(accessSystem='http') # the accessSystem argument is not really - # needed, since "http" is the default. - i.set_proxy('http://localhost:8080/') - - You can force a direct connection to the net setting the proxy - to a null value (i.e.: i.set_proxy('')). - - - FEATURES OF THE SQL DATA ACCESS SYSTEM - ====================================== - -* Returns every information available in the plain text data files. -* Every database supported by SQLObject and SQLAlchemy is available. - - - FEATURES OF THE MOBILE DATA ACCESS SYSTEM - ========================================= - -* Very lightweight, returns almost every needed information. -* Accessories data sets (like 'goofs', 'trivia' and so on) are always - available (being a subclass of the 'http' data access system). - - diff -pruN 5.1-1/docs/README.unicode 6.6-1/docs/README.unicode --- 5.1-1/docs/README.unicode 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.unicode 1970-01-01 00:00:00.000000000 +0000 @@ -1,164 +0,0 @@ - UNICODE SUPPORT - =============== - -Starting with release 2.4, IMDbPY internally manages (almost) every string -using unicode, with UTF-8 encoding. -Since release 3.0, every string containing some sort of information is -guarantee to be unicode (notable exceptions are dictionary keywords and -movieID/personID, where they are stored as strings). - -The good: we can correctly manage "foreign" names, titles and other - information. - Previously every string was stored in bytecode, losing information - about the original charset. - Without knowing the charset, how can you know that the bytecode - string 'Lina Wertm\xfcller' is west-European iso-8859-1 (and so - it's "Lina Wertmüller" - if you're reading this file as UTF-8) - and not Cyrillic KOI-8-R (resulting in "Lina WertmЭller")? - Using unicode, you can store every human language, and show/print - every char correctly, provided that your local charset (and font) - is right. - -The bad: in primis, performances will suffer: IMDbPY does _a lot_ (and - with _a lot_ I mean _A BLOODY DAMN LOT_) of string operations - (moving, copying, splitting, searching, slicing, ...) and moving - to unicode the slow down will be measurable (and probably - noticeable). - Moreover, every IMDbPY-base program will need to be modified, - because utf-8 chars must be encoded-back to your local charset - before they can be printed on screen or on files. - -The ugly: converting to unicode a program so huge, born without unicode - support from start, is prone to errors, bugs, spontaneous - combustion and eternal damnation! - You can't mix bytecode strings (with unknown charset) and unicode - with impunity: an exception will be raised because python - doesn't know the encoding of the bytecode string, that must be - explicitly specified. - - - INPUT - ===== - -Searching for a movie title or a person name, you (or another program) -should pass a unicode string. Since input and output are always -encoded (using some scheme for representing characters using bytes), -you'll typically need to decode the input into a Python unicode -string. - -The Python builtin function 'unicode' takes an encoded string and the -name of the charset it's encoded in and returns a Python unicode -string. - -E.g., you're writing on a terminal with iso-8859-1 charset (aka latin-1): - >>> from imdb import IMDb - >>> ia = IMDb() - >>> - >>> lat1_str = 'Lina Wertm�ler' # written on a latin-1 terminal - >>> unicode_str = unicode(lat1_str, 'iso-8859-1') - >>> - >>> results = ia.search_person(unicode_str) - -If you pass a non-Unicode string to search_person(), search_movie() or -search_episode() functions, IMDbPY attempts to guess the encoding, -using the sys.stdin.encoding or the value returned from the -sys.getdefaultencoding function. Trust me: you want to provide a -Unicode string... - -Maybe in a future release the IMDb() function can take a "defaultInputEncoding" -argument or something. - - - OUTPUT - ====== - -You've searched for a person or a movie, you've retrieved the information you -wanted. Cool. Now you're about to print these information to the screen, -or send it to a file or over a network socket. Ok, wait a minute. - -Before you proceed, you need to encode the unicode chars to strings -in the charset you will use to display/save/send it: - >>> from imdb import IMDb - >>> ia = IMDb() - >>> - >>> gmv_str = unicode('gian maria volonte', 'ascii') # optional, IT'S ascii... - >>> gmv = ia.search_person(gmv_str)[0] - >>> ia.update(gmv) # fetch the default set of information. - >>> - >>> gmv['name'] - u'Gian Maria Volont\xe9' - >>> - >>> type(gmv['name']) - >>> - >>> - >>> print gmv['name'] # WRONG: because if you are on an ASCII only terminal... - Traceback (most recent call last): - File "", line 1, in ? - UnicodeEncodeError: 'ascii' codec can't encode character u'\xe9' in position 17: ordinal not in range(128) - >>> - >>> print gmv['name'].encode(yourLocalEncoding, 'replace') # CORRECT. - Gian Maria Volonté - - -You have to use the encode() method of unicode strings to obtain a string -suited for your local configuration. -The encoding depends on your system and on what you've to do with these -strings. -The second (optional) argument of the encode() method specifies what -to do with the unicode chars that cannot be represented in the encoding -of your choice. If not specified, a UnicodeEncodeError exception is -raised, so be prepared. -Other values are 'ignore' to skip these chars, 'replace' to substitute -these chars with question marks ('?'), 'xmlcharrefreplace' to replace -the chars with XML references (e.g.: "é" for "é"). - -If at all possible, you're much better off using an encoding like 'utf-8', -that can handle any unicode char, than older encodings like ASCII -and Latin-1. - - - WRITING IMDbPY-based PROGRAMS - ============================= - -In the imdb.helpers module you can find some functions useful to -manage/translate unicode strings in some common situations. - - - RULE OF THUMB - ============= - -Always convert to/from unicode at the I/O level: at the first moment -you've got some strings from the user (terminal) or the net (sockets, -web forms, whatever). You need to know the encoding of the input, -checking sys.stding.encoding, the LANG/LC_* environment variables, -the headers of the http request and so on. -Whenever you're outputting information about movies or persons, -convert these unicode string to bytecode strings using the encoding -of your output channel (terminal, net, web pages, ...) - -Remember: "u = unicode(string, inputEncoding)" convert your encoded input - string to unicode, - - "s = u.encode(outputEncoding, manageErrors)" convert unicode - strings to strings encoded for your local environment. - - - LINKS - ===== - -* The Absolute Minimum Every Software Developer Absolutely, Positively Must - Know About Unicode and Character Sets (No Excuses!): - http://www.joelonsoftware.com/articles/Unicode.html - -* Python Unicode HOWTO: - http://www.amk.ca/python/howto/unicode - -* Dive Into Python, unicode page: - http://diveintopython.org/xml_processing/unicode.html - -* How to Use UTF-8 with Python: - http://evanjones.ca/python-utf8.html - -* End to End Unicode Web Applications in Python: - http://dalchemy.com/opensource/unicodedoc/ - diff -pruN 5.1-1/docs/README.users 6.6-1/docs/README.users --- 5.1-1/docs/README.users 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/README.users 1970-01-01 00:00:00.000000000 +0000 @@ -1,105 +0,0 @@ - IMDbPY FOR THE USERS - ==================== - -As I've already said, IMDbPY by itself is not really useful if you're -not a developer; anyway, some simple example scripts are included in -the ./bin directory. -You can find other IMDbPY-based programs here: - http://imdbpy.sf.net/?page=programs - -The 'search_movie.py' script takes a single argument, which must be -a title of a movie to search; if the title contains spaces or -other "strange" chars, enclose the title in single or double quotes. -It will print a list of imdbID and long imdb titles. - -The 'get_movie.py' script takes a single argument: a movieID -and it will print some information about the movie. -The movieID is a unique identifier for a single movie. -Notice that, since IMDbPY can take its information from different -sources, the movieID can take different shapes. -By default IMDbPY accesses the web using its 'http' data access -system, and so in this case movieID are the same imdbIDs that you -can find on the web page for that movie (the same is true using the -'mobile' data access system). -On the other hand, if you've configured IMDbPY to use the 'sql' -data access system, the movieIDs are still unique, but totally -arbitrary and not related to the imdbIDs used by the web server; this, -because the list of imdbIDs is not published with the plain text data -files. Also notice that these movieIDs generated by 'sql', are only -valid for your current setup, and will change when you'll update your -database with a new set of plain text data files. - -The 'get_first_movie.py' works like 'search_movie.py', but it will only print -information about the first matching title. - -Now guess what 'get_person.py', 'search_person.py','get_first_person.py', -'get_character.py', 'search_character.py','get_first_character.py', -'get_company.py', 'search_company.py', 'get_first_company.py', -'search_keyword.py', 'get_keyword.py' and 'get_top_bottom_movies.py' -scripts do... :-) - -Remember that you've to be connected to the net, if you want to -use these scripts! -If the HTTP_PROXY environment variable is set, the referred proxy -is used. - -Take a look at the scripts; they're somewhat commented; maybe -you can customize it... - - -Examples: - -$ search_movie.py 'the passion' - 20 results for "the passion": -Passion of the Christ, The (2004) -Passion, The (2003/I) -Passion, The (1999) (TV) -Patima (1975) -Andrei Rublyov (1969) -Passion de Jeanne d'Arc, La (1928) -Passion of Darkly Noon, The (1996) -Passion of Ayn Rand, The (1999) -Passion Béatrice, La (1987) -Passion, En (1969) -Pride and the Passion, The (1957) -"Charles II: The Power & the Passion" (2003) (mini) -Pasión de María Elena, La (2003) -Pasión según Berenice, La (1976) -Passion of Rita Camilleri, The (1993) -Culture, Water, Money: The Passion of the Frontier (1998) -Sanguisuga conduce la danza, La (1975) -Passion of John Ruskin, The (1994) -Making of 'The Passion of the Christ', The (2004) (TV) -Scream of the Butterfly (1965) - - -# Here we assumes you've not configured IMDbPY to use a local SQL database, -# and so we use '0133093' as the movieID. -$ get_movie.py 0133093 -Movie -===== -Title: Matrix, The -Genres: Action, Thriller, Sci-Fi. -Director: Andy Wachowski (as The Wachowski Brothers), Larry Wachowski (as The Wachowski Brothers). -Writer: Andy Wachowski (written by) (as The Wachowski Brothers), Larry Wachowski (written by) (as The Wachowski Brothers). -Cast: Keanu Reeves (Neo (Thomas A. Anderson)), Laurence Fishburne (Morpheus), Carrie-Anne Moss (Trinity), Hugo Weaving (Agent Smith), Joe Pantoliano (Cypher (Mr. Reagan)). -Runtime: 136. -Country: USA. -Language: English. -Rating: 8.5 -Votes: 114,264 -Plot: In the near future, a computer hacker named Neo (Keanu Reeves) discovers that all life on Earth may be nothing more than an elaborate facade created by a malevolent cyber-intelligence, for the purpose of placating us while our life essence is "farmed" to fuel the Matrix's campaign of domination in the "real" world. He joins like-minded Rebel warriors Morpheus (Laurence Fishburne) and Trinity (Carrie Ann Moss) in their struggle to overthrow the Matrix. - - -$ get_first_character.py 'Jesse James' - Best match for "Jesse James" -Character -===== -Name: Jesse James -Biography: History::Born: September 5, 1847 in Clay County, Missouri, USA - -Died: April 3, 1882 in St. Joseph, Missouri, USA - - [...] - - diff -pruN 5.1-1/docs/TODO.txt 6.6-1/docs/TODO.txt --- 5.1-1/docs/TODO.txt 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/docs/TODO.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,142 +0,0 @@ - TODO for IMDbPY - =============== - -See the code, and search for XXX, FIXME and TODO. - -NOTE: it's always time to clean the code! - - -[general] -* improve the logging facility. -* ability to silent warnings/logging. -* create mobile versions for smart phones (with GUI). -* Write better summary() methods for Movie, Person, Character - and Company classes. -* Some portions of code are poorly commented. -* The documentation is written in my funny Anglo-Bolognese. -* a better test-suite is really needed. -* Compatibility with Python 2.2 and previous versions is no more assured - for every data access system (the imdbpy2sql.py script for sure - requires at least Python 2.3). Be sure to keep 2.2 compatibility - at least for 'http' and 'mobile', since they are used by mobile devices. -* The analyze_title/build_title functions are grown too complex and - beyond their initial goals. - - -[searches] -* Support advanced query for movie titles and person/character/company names - - if possible this should be available in every data access systems. - - -[xml] -* bugs of the helpers.parseXML function: - * doesn't handle Movie instances as keys of a dictionary; person->episodes, - character->episodes, character->quotes. - * changes tuples to lists (not a big deal). - * person/movieRefs are lost. - * uses only BeautifulSoup; maybe it can work with lxml, too. -* use the ID type, instead of CDATA, for the 'id' attribute? -* keywords (and hence tags) generated by SQL may be slightly different, - and must be integrate in the DTD. - - -[Movie objects] -* Define invariable names for the sections (the keys you use to access - info stored in a Movie object). -* Should a movie object automatically build a Movie object, when - an 'episode of' dictionary is in the data? -* Should isSameTitle() check first the accessSystem and the movieID, - and use 'title' only if movieID is None? -* For TV series the list of directors/writers returned by 'sql' - is a long list with every single episodes listed in the - 'notes' attribute (i.e.: the same person is listed more than one time, - just with a different note). - For 'http' and 'mobile' there's a list with one item for every - person, with a long 'notes' listing every episode. - It's not easy to split these information since they can contain - notes ("written by", "as Aka Name", ...) -* The 'laserdisc' information for 'sql' is probabily - wrong: I think they merge data from different laserdisc titles. - Anyway these data are no more updated by IMDb, and so... -* there are links to hollywoodreporter.com that are not gathered in - the "external reviews" page. - - -[Person objects] -* Define invariable names for the sections (the keys you use to access - info stored in a Person object). -* Should isSameName() check first the accessSystem and the personID, - and use 'name' only if personID is None? -* Fetching data from the web ('http' and 'mobile'), the filmography - for a given person contains a list named "himself" or "herself" for - movies/shows where they were not acting. - In 'sql', these movies/shows are listed in the "actor" or "actress" list. - Check if this is still true, with the new IMDb's schema. - - -[Character objects] -* Define invariable names for the sections (the keys you use to access - info stored in a Character object). - - -[Company objects] -* Define invariable names for the sections (the keys you use to access - info stored in a Company object). - - -[http data access system] -* Serious confusion about handling XML/HTML/SGML char references; there - are too many fixes for special cases, and a better understanding of how - lxml and BeautifulSoup behave is required. -* If the access through the proxy fails, is it possible to - automatically try without? It doesn't seem easy... -* Access to the "my IMDb" functions for registered users would - be really cool. -* Gather more movies' data: user comments, laserdisc details, trailers, - posters, photo gallery, on tv, schedule links, showtimes, message boards. -* Gather more people's data: photo gallery. - - -[httpThin data access system] -* It should be made _really_ faster than 'http'. - - -[mobile data access system] -* General optimization. -* Make find() methods case insensitive. - - -[sql data access system] -NOTE NOTE NOTE: this is still beta code and I'm not a database guru; -moreover I'm short of time and so I will be happy to fix every bug -you'll find, but if you're about to write me an email like "ehi, -the database access should be faster", "the imdbpy2sql.py script must -run with 64 MB of RAM and complete in 2 minutes" or "your database -layout sucks: I've an idea for a better structure...", well, consider -that _these_ kinds of email will be probably immediately discarded. - -I _know_ these are important issues, but I've neither the time nor -the ability to fix these problems by myself, sorry. -Obviously if you want to contribute with patches, new code, new -SQL queries and new database structures you're welcome and I will -be very grateful for your work. - -Again: if there's something that bother you, write some code. -It's free software, after all. - -Things to do: -* The imdbpy2sql.py script MUST be run on a database with empty tables; - unfortunately so far a SQL installation can't be "updated" without - recreating the database from scratch. - IMDb releases also "diff" files to keep the plain text files updated; - it would be wonderful to directly use these diff files to upgrade the - SQL database, but I think this is a nearly impossible task. - A lot of attempts were made in this direction, always failing. -* There are a lot of things to do to improve SQLAlchemy support (especially - in terms of performances); see FIXME/TODO/XXX notices in the code. -* The pysqlite2.dbapi2.OperationalError exception is raise when SQLite - is used with SQLAlchemy (but only if the --sqlite-transactions command - line argument is used). -* With 0.5 branch of SQLAlchemy, it seems that there are serious problems - using SQLite; try switching to SQLObject, as a temporary solution. - diff -pruN 5.1-1/docs/usage/access.rst 6.6-1/docs/usage/access.rst --- 5.1-1/docs/usage/access.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/access.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,42 @@ +.. _access: + +Access systems +============== + +IMDbPY supports different ways of accessing the IMDb data: + +- Fetching data directly from the web server. + +- Getting the data from a SQL database that can be created from + the downloadable data sets provided by the IMDb. + ++------------------+-------------+----------------------+ +| access system | aliases | data source | ++==================+=============+======================+ +| (default) 'http' | 'https' | imdb.com web server | +| | | | +| | 'web' | | +| | | | +| | 'html' | | ++------------------+-------------+----------------------+ +| 's3' | 's3dataset' | downloadable dataset | +| | | | +| | | *after Dec 2017* | ++------------------+-------------+----------------------+ +| 'sql' | 'db' | downloadable dataset | +| | | | +| | 'database' | *until Dec 2017* | ++------------------+-------------+----------------------+ + +.. note:: + + Since release 3.4, the :file:`imdbpy.cfg` configuration file is available, + so that you can set a system-wide (or per-user) default. The file is + commented with indication of the location where it can be put, + and how to modify it. + + If no :file:`imdbpy.cfg` file is found (or is not readable or + it can't be parsed), 'http' will be used the default. + +See the :ref:`s3` and :ref:`ptdf` documents for more information about +SQL based access systems. diff -pruN 5.1-1/docs/usage/adult.rst 6.6-1/docs/usage/adult.rst --- 5.1-1/docs/usage/adult.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/adult.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,52 @@ +Adult movies +============ + +*IMDbPY for (too) sensitive people.* + +Since version 2.0 (shame on me! I've noticed this only after more than a year +of development!!!) adult movies are included by default in search results. + +If for some unintelligible reason you don't want classics like +"Debbie Does Dallas" to show up in your searches, you can disable this feature +by initializing the :class:`IMDb ` class with the ``adultSearch`` +argument set to ``False``: + +.. code-block:: python + + >>> ia = IMDb(accessSystem='http', adultSearch=False) + + +This behavior can also be modified at runtime by calling the +:meth:`do_adult_search ` method: + +.. code-block:: python + + >>> ia = IMDb(accessSystem='http') # by default in horny mode + >>> movies = ia.search_movie('debby does dallas', results=5) + >>> movieIDs = [m.movieID for m in movies] + >>> '0077415' in movieIDs # Debbie Does Dallas (1978) + True + >>> ia.do_adult_search(False) # switch to puritan behavior + >>> movies = ia.search_movie('debby does dallas', results=5) + >>> movieIDs = [m.movieID for m in movies] + >>> '0077415' in movieIDs # Debbie Does Dallas (1978) + False + + +The :meth:`do_adult_search ` +method of the HTTP data access system can take two more arguments: +``cookie_id`` and ``cookie_uu``, so that you can select *your own* +IMDb account; if cookie id is set to None, no cookies are sent. +These parameters can also be set in the :file:`imdbpy.cfg` configuration file. +To find the strings you need to use, see your "cookie" or "cookie.txt" files. +Obviously, you'll need to activate the "adult movies" option for your account; +see http://imdb.com/find/preferences?_adult=1 + +Since version 2.2 all data access systems (sql) support +the same behavior (i.e.: you can set the ``adultSearch`` argument and +use the ``do_adult_search`` method). + +Note that for the sql data access system, only results from +the ``search_movie()`` and ``search_episode()`` methods are filtered: +there's no easy (and fast) way to tell whether an actor/actress +is a porn star or not. diff -pruN 5.1-1/docs/usage/character.rst 6.6-1/docs/usage/character.rst --- 5.1-1/docs/usage/character.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/character.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,25 @@ +:orphan: + +Characters +========== + +It works mostly like the Person class. :-) + +For more information about the "currentRole" attribute, see the +README.currentRole file. + + +Character associated to a person who starred in a movie, and its notes: + +.. code-block:: python + + person_in_cast = movie['cast'][0] + notes = person_in_cast.notes + character = person_in_cast.currentRole + +Check whether a person worked in a given movie or not: + +.. code-block:: python + + person in movie + movie in person diff -pruN 5.1-1/docs/usage/company.rst 6.6-1/docs/usage/company.rst --- 5.1-1/docs/usage/company.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/company.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,12 @@ +:orphan: + +Companies +========= + +It works mostly like the Person class. :-) + +The "currentRole" attribute is always None. + + +As for Person/Character and Movie objects, you can test -using the "in" +operator- if a Company has worked on a given Movie. diff -pruN 5.1-1/docs/usage/data-interface.rst 6.6-1/docs/usage/data-interface.rst --- 5.1-1/docs/usage/data-interface.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/data-interface.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,185 @@ +Data interface +============== + +The IMDbPY objects that represent movies, people and companies provide +a dictionary-like interface where the key identifies the information +you want to get out of the object. + +At this point, I have really bad news: what the keys are is a little unclear! + +In general, the key is the label of the section as used by the IMDb web server +to present the data. If the information is grouped into subsections, +such as cast members, certifications, distributor companies, etc., +the subsection label in the HTML page is used as the key. + +The key is almost always lowercase; underscores and dashes are replaced +with spaces. Some keys aren't taken from the HTML page, but are defined +within the respective class. + + +Information sets +---------------- + +IMDbPY can retrieve almost every piece of information of a movie, person or +company. This can be a problem, because (at least for the "http" data access +system) it means that a lot of web pages must be fetched and parsed. +This can be both time- and bandwidth-consuming, especially if you're interested +in only a small part of the information. + +The :meth:`get_movie `, +:meth:`get_person ` and +:meth:`get_company ` methods take an optional +``info`` parameter, which can be used to specify the kinds of data to fetch. +Each group of data that gets fetched together is called an "information set". + +Different types of objects have their own available information sets. +For example, the movie objects have a set called "vote details" for +the number of votes and their demographic breakdowns, whereas person objects +have a set called "other works" for miscellaneous works of the person. +Available information sets for each object type can be queried +using the access object: + +.. code-block:: python + + >>> from imdb import IMDb + >>> ia = IMDb() + >>> ia.get_movie_infoset() + ['airing', 'akas', ..., 'video clips', 'vote details'] + >>> ia.get_person_infoset() + ['awards', 'biography', ..., 'other works', 'publicity'] + >>> ia.get_company_infoset() + ['main'] + +For each object type, only the important information will be retrieved +by default: + +- for a movie: "main", "plot" +- for a person: "main", "filmography", "biography" +- for a company: "main" + +These defaults can be retrieved from the ``default_info`` attributes +of the classes: + +.. code-block:: python + + >>> from imdb.Person import Person + >>> Person.default_info + ('main', 'filmography', 'biography') + +Each instance also has a ``current_info`` attribute for tracking +the information sets that have already been retrieved: + +.. code-block:: python + + >>> movie = ia.get_movie('0133093') + >>> movie.current_info + ['main', 'plot', 'synopsis'] + +The list of retrieved information sets and the keys they provide can be +taken from the ``infoset2keys`` attribute: + +.. code-block:: python + + >>> movie = ia.get_movie('0133093') + >>> movie.infoset2keys + {'main': ['cast', 'genres', ..., 'top 250 rank'], 'plot': ['plot', 'synopsis']} + >>> movie = ia.get_movie('0094226', info=['taglines', 'plot']) + >>> movie.infoset2keys + {'taglines': ['taglines'], 'plot': ['plot', 'synopsis']} + >>> movie.get('title') + >>> movie.get('taglines')[0] + 'The Chicago Dream is that big' + +Search operations retrieve a fixed set of data and don't have the concept +of information sets. Therefore objects listed in searches will have even less +information than the defaults. For example, if you do a movie search operation, +the movie objects in the result won't have many of the keys that would be +available on a movie get operation: + +.. code-block:: python + + >>> movies = ia.search_movie('matrix') + >>> movie = movies[0] + >>> movie + + >>> movie.current_info + [] + >>> 'genres' in movie + False + +Once an object is retrieved (through a get or a search), its data can be +updated using the :meth:`update ` method with the desired +information sets. Continuing from the example above: + +.. code-block:: python + + >>> 'median' in movie + False + >>> ia.update(movie, info=['taglines', 'vote details']) + >>> movie.current_info + ['taglines', 'vote details'] + >>> movie['median'] + 9 + >>> ia.update(movie, info=['plot']) + >>> movie.current_info + ['taglines', 'vote details', 'plot', 'synopsis'] + +Beware that the information sets vary between access systems: +locally not every piece of data is accessible, whereas -for example for SQL- +accessing one set of data means automatically accessing a number of other +information (without major performance drawbacks). + + +Composite data +-------------- + +In some data, the (not-so) universal ``::`` separator is used to delimit +parts of the data inside a string, like the plot of a movie and its author: + +.. code-block:: python + + >>> movie = ia.get_movie('0094226') + >>> plot = movie['plot'][0] + >>> plot + "1920's prohibition ... way to get him.::Jeremy Perkins " + +As a rule, there's at most one such separator inside a string. Splitting +the string will result in two logical pieces as in ``TEXT::NOTE``. +The :func:`imdb.helpers.makeTextNotes` function can be used to create a custom +function to pretty-print this kind of information. + + +References +---------- + +Sometimes the collected data contains strings with references to other movies +or persons, e.g. in the plot of a movie or the biography of a person. +These references are stored in the Movie, Person, and Character instances; +in the strings you will find values like _A Movie (2003)_ (qv) +or 'A Person' (qv) or '#A Character# (qv)'. When these strings are accessed +(like movie['plot'] or person['biography']), they will be modified using +a provided function, which must take the string and two dictionaries +containing titles and names references as parameters. + +By default the (qv) strings are converted in the "normal" format +("A Movie (2003)", "A Person" and "A Character"). + +You can find some examples of these functions in the +imdb.utils module. + +The function used to modify the strings can be set with the ``defaultModFunct`` +parameter of the IMDb class or with the ``modFunct`` parameter +of the ``get_movie``, ``get_person``, and ``get_character`` methods: + +.. code-block:: python + + import imdb + i = imdb.IMDb(defaultModFunct=imdb.utils.modHtmlLinks) + +or: + +.. code-block:: python + + import imdb + i = imdb.IMDb() + i.get_person('0000154', modFunct=imdb.utils.modHtmlLinks) diff -pruN 5.1-1/docs/usage/index.rst 6.6-1/docs/usage/index.rst --- 5.1-1/docs/usage/index.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/index.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,25 @@ +Usage +===== + +Here you can find information about how you can use IMDbPY in your own +programs. + +.. warning:: + + This document is far from complete: the code is the final documentation! ;-) + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + quickstart + data-interface + role + series + adult + info2xml + l10n + access + s3 + ptdf diff -pruN 5.1-1/docs/usage/info2xml.rst 6.6-1/docs/usage/info2xml.rst --- 5.1-1/docs/usage/info2xml.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/info2xml.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,106 @@ +Information in XML format +========================= + +Since version 4.0, IMDbPY can output information of Movie, Person, Character, +and Company instances in XML format. It's possible to get a single information +(a key) in XML format, using the ``getAsXML(key)`` method (it will return None +if the key is not found). E.g.: + +.. code-block:: python + + from imdb import IMDb + ia = IMDb('http') + movie = ia.get_movie(theMovieID) + print(movie.getAsXML('keywords')) + +It's also possible to get a representation of a whole object, using +the ``asXML()`` method:: + + print(movie.asXML()) + +The ``_with_add_keys`` argument of the ``asXML()`` method can be set +to False (default: True) to exclude the dynamically generated keys +(like 'smart canonical title' and so on). + + +XML format +---------- + +Keywords are converted to tags, items in lists are enclosed in +a 'item' tag, e.g.: + +.. code-block:: xml + + + a keyword + another keyword + + +Except when keys are known to be not fixed (e.g.: a list of keywords), +in which case this schema is used: + +.. code-block:: xml + + + ... + + +In general, the 'key' attribute is present whenever the used tag doesn't match +the key name. + +Movie, Person, Character and Company instances are converted as follows +(portions in square brackets are optional): + +.. code-block:: xml + + + A Long IMDb Movie Title (YEAR) + [ + + Name Surname + [A Note About The Person] + + ] + [A Note About The Movie] + + +Every 'id' can be empty. + +The returned XML string is mostly not pretty-printed. + + +References +---------- + +Some text keys can contain references to other movies, persons and characters. +The user can provide the ``defaultModFunct`` function (see +the "MOVIE TITLES AND PERSON/CHARACTER NAMES REFERENCES" section of +the README.package file), to replace these references with their own strings +(e.g.: a link to a web page); it's up to the user, to be sure +that the output of the defaultModFunct function is valid XML. + + +DTD +--- + +Since version 4.1 a DTD is available; it can be found in this +directory or on the web, at: http://imdbpy.sf.net/dtd/imdbpy41.dtd + +The version number changes with the IMDbPY version. + + +Localization +------------ + +Since version 4.1 it's possible to translate the XML tags; +see README.locale. + + +Deserializing +------------- + +Since version 4.6, you can dump the generated XML in a string or +in a file, using it -later- to rebuild the original object. +In the ``imdb.helpers`` module there's the ``parseXML()`` function which +takes a string as input and returns -if possible- an instance of the Movie, +Person, Character or Company class. diff -pruN 5.1-1/docs/usage/l10n.rst 6.6-1/docs/usage/l10n.rst --- 5.1-1/docs/usage/l10n.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/l10n.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,79 @@ +Localization +============ + +Since version 4.1 the labels that describe the information are translatable. + +.. admonition:: Limitation + + Internal messages or exceptions are not translatable, + the internationalization is limited to the "tags" returned by + the ``getAsXML`` and ``asXML`` methods of the Movie, Person, Character, + or Company classes. + + Beware that in many cases these "tags" are not the same as the "keys" used + to access information in the same class. For example, you can translate + the tag "long-imdb-name" -the tag returned by the call + ``person.getAsXML('long imdb name')``, but not the key "long imdb name" + itself. To translate keys, you can use + the :func:`helpers.translateKey ` function. + +If you want to add i18n to your IMDbPY-based application, all you need to do +is to switch to the ``imdbpy`` text domain: + +.. code-block:: python + + >>> import imdb.locale + >>> import gettext + >>> gettext.textdomain('imdbpy') + 'imdbpy' + >>> from gettext import gettext as _ + >>> _('art-department') + 'Art department' + >>> import os + >>> os.environ['LANG'] = 'it_IT' + >>> _('art-department') + 'Dipartimento artistico' + +If you want to translate IMDbPY into another language, see +the :ref:`translate` document for instructions. + + +Articles in titles +------------------ + +To convert a title to its canonical format as in "Title, The", IMDbPY makes +some assumptions about what is an article and what isn't, and this can lead +to some wrong canonical titles. For example, it can canonicalize the title +"Die Hard" as "Hard, Die" because it guesses "Die" as an article (and it is, +in Germany...). + +To solve this problem, there are other keys: "smart canonical title", +"smart long imdb canonical title", "smart canonical series title", +"smart canonical episode title" which can be used to do a better job +converting a title into its canonical format. + +This works, but it needs to know about articles in various languages: +if you want to help, see the :attr:`linguistics.LANG_ARTICLES` and +:attr:`linguistics.LANG_COUNTRIES` dictionaries. + +To guess the language of a movie title, call its 'guessLanguage' method +(it will return None, if unable to guess). +If you want to force a given language instead of the guessed one, you +can call its 'smartCanonicalTitle' method, setting the 'lang' argument +appropriately. + +Alternative titles +------------------ + +Sometimes it's useful to manage a title's alternatives (AKAs) knowing +their languages. In the 'helpers' module there are some (hopefully) +useful functions: + +- ``akasLanguages(movie)`` - Given a movie, return a list of tuples + in (lang, AKA) format (lang can be None, if unable to detect). + +- ``sortAKAsBySimilarity(movie, title)`` - Sort the AKAs on a movie considering + how much they are similar to a given title (see the code for more options). + +- ``getAKAsInLanguage(movie, lang)`` - Return a list of AKAs of the movie + in the given language (see the code for more options). diff -pruN 5.1-1/docs/usage/movie.rst 6.6-1/docs/usage/movie.rst --- 5.1-1/docs/usage/movie.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/movie.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,126 @@ +:orphan: + +Movies +====== + +Below is a list of each main key, the type of its value, and a short +description or an example: + +title (string) + The "usual" title of the movie, like "The Untouchables". + +long imdb title (string) + "Uncommon Valor (1983/II) (TV)" + +canonical title (string) + The title in canonical format, like "Untouchables, The". + +long imdb canonical title (string) + "Patriot, The (2000)" + +year (string) + The release year, or '????' if unknown. + +kind (string) + One of: 'movie', 'tv series', 'tv mini series', 'video game', 'video movie', + 'tv movie', 'episode' + +imdbIndex (string) + The roman numeral for movies with the same title/year. + +director (Person list) + A list of directors' names, e.g.: ['Brian De Palma']. + +cast (Person list) + A list of actors/actresses, with the currentRole instance variable + set to a Character object which describe his role. + +cover url (string) + The link to the image of the poster. + +writer (Person list) + A list of writers, e.g.: ['Oscar Fraley (novel)']. + +plot (list) + A list of plot summaries and their authors. + +rating (string) + User rating on IMDb from 1 to 10, e.g. '7.8'. + +votes (string) + Number of votes, e.g. '24,101'. + +runtimes (string list) + List of runtimes in minutes ['119'], or something like ['USA:118', 'UK:116']. + +number of episodes (int) + Number or episodes for a TV series. + +color info (string list) + ["Color (Technicolor)"] + +countries (string list) + Production's country, e.g. ['USA', 'Italy']. + +genres (string list) + One or more of: Action, Adventure, Adult, Animation, Comedy, Crime, + Documentary, Drama, Family, Fantasy, Film-Noir, Horror, Musical, Mystery, + Romance, Sci-Fi, Short, Thriller, War, Western, and other genres + defined by IMDb. + +akas (string list) + List of alternative titles. + +languages (string list) + A list of languages. + +certificates (string list) + ['UK:15', 'USA:R'] + +mpaa (string) + The MPAA rating. + +episodes (series only) (dictionary of dictionaries) + One key for every season, one key for every episode in the season. + +number of episodes (series only) (int) + Total number of episodes. + +number of seasons (series only) (int) + Total number of seasons. + +series years (series only) (string) + Range of years when the series was produced. + +episode of (episodes only) (Movie object) + The series to which the episode belongs. + +season (episodes only) (int) + The season number. + +episode (episodes only) (int) + The number of the episode in the season. + +long imdb episode title (episodes only) (string) + Episode and series title. + +series title (string) + The title of the series to which the episode belongs. + +canonical series title (string) + The canonical title of the series to which the episode belongs. + + +Other keys that contain a list of Person objects are: costume designer, +sound crew, crewmembers, editor, production manager, visual effects, +assistant director, art department, composer, art director, cinematographer, +make up, stunt performer, producer, set decorator, production designer. + +Other keys that contain list of companies are: production companies, special +effects, sound mix, special effects companies, miscellaneous companies, +distributors. + +Converting a title to its "Title, The" canonical format, IMDbPY makes +some assumptions about what is an article and what isn't, and this could +lead to some wrong canonical titles. For more information on this subject, +see the "ARTICLES IN TITLES" section of the README.locale file. diff -pruN 5.1-1/docs/usage/person.rst 6.6-1/docs/usage/person.rst --- 5.1-1/docs/usage/person.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/person.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,27 @@ +:orphan: + +Persons +======= + +It works mostly like the Movie class. :-) + +The Movie class defines a ``__contains__()`` method, which is used to check +if a given person has worked in a given movie with the syntax: + +.. code-block:: python + + if personObject in movieObject: + print('%s worked in %s' % (personObject['name'], movieObject['title'])) + +The Person class defines a ``isSamePerson(otherPersonObject)`` method, which +can be used to compare two person objects. This can be used to check whether +an object has retrieved complete information or not, as in the case of a Person +object returned by a query: + +.. code-block:: python + + if personObject.isSamePerson(otherPersonObject): + print('they are the same person!') + +A similar method is defined for the Movie class, and it's called +``isSameTitle(otherMovieObject)``. diff -pruN 5.1-1/docs/usage/ptdf.rst 6.6-1/docs/usage/ptdf.rst --- 5.1-1/docs/usage/ptdf.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/ptdf.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,357 @@ +.. _ptdf: + +Old data files +============== + +.. warning:: + + Since the end of 2017, IMDb is no longer updating the data files which are + described in this document. For working with the updated + -but less comprehensive- downloadable data, check the :ref:`s3` document. + +Until the end of 2017, IMDb used to distribute some of its data as downloadable +text files. IMDbPY can import this data into a database and make it +accessible through its API. + +For this, you will first need to install `SQLAlchemy`_ and the libraries +that are needed for the database server you want to use. Check out +the `SQLAlchemy dialects`_ documentation for more detail. + +Then, follow these steps: + +#. Download the files from the following address and put all of them + in the same directory: + ftp://ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/frozendata/ + + You can just download the files you need instead of downloading all files. + The files that are not downloaded will be skipped during import. + This feature is still quite untested, so please report any bugs. + + .. warning:: + + Beware that the :file:`diffs` subdirectory contains + **a lot** of files you **don't** need, so don't start mirroring + everything! + +#. Create a database. Use a collation like ``utf8_unicode_ci``. + +#. Import the data using the :file:`imdbpy2sql.py` script:: + + imdbpy2sql.py -d /path/to/the/data_files_dir/ -u URI + + *URI* is the identifier used to access the SQL database. For example:: + + imdbpy2sql.py -d ~/Download/imdb-frozendata/ \ + -u postgres://user:password@localhost/imdb + +Once the import is finished, you will have a SQL database with all +the information and you can use the normal IMDbPY API: + +.. code-block:: python + + from imdb import IMDb + + ia = IMDb('sql', uri='postgres://user:password@localhost/imdb') + + results = ia.search_movie('the matrix') + for result in results: + print(result.movieID, result) + + matrix = results[0] + ia.update(matrix) + print(matrix.keys()) + + +.. note:: + + It should be noted that the :file:`imdbpy2sql.py` script will not create + any foreign keys, but only indexes. If you need foreign keys, try using + the version in the "imdbpy-legacy" branch. + + If you need instructions on how to manually build the foreign keys, + see `this comment by Andrew D Bate`_. + + +Performance +----------- + +The import performance hugely depends on the underlying module used to access +the database. The :file:`imdbpy2sql.py` script has a number of command line +arguments for choosing presets that can improve performance in specific +database servers. + +The fastest database appears to be MySQL, with about 200 minutes to complete +on my test system (read below). A lot of memory (RAM or swap space) +is required, in the range of at least 250/500 megabytes (plus more +for the database server). In the end, the database requires between +2.5GB and 5GB of disk space. + +As said, the performance varies greatly using one database server or another. +MySQL, for instance, has an ``executemany()`` method of the cursor object +that accepts multiple data insertion with a single SQL statement; other +databases require a call to the ``execute()`` method for every single row +of data, and they will be much slower -2 to 7 times slower than MySQL. + +There are generic suggestions that can lead to better performance, such as +turning off your filesystem journaling (so it can be a good idea to remount +an ext3 filesystem as ext2 for example). Another option is using +a ramdisk/tmpfs, if you have enough RAM. Obviously these have effect only at +insert-time; during day-to-day use, you can turn journaling on again. +You can also consider using CSV output as explained below, if your database +server can import CSV files. + +I've done some tests, using an AMD Athlon 1800+, 1GB of RAM, over a complete +plain text data files set (as of 11 Apr 2008, with more than 1.200.000 titles +and over 2.200.000 names): + ++----------------------+------------------------------------------------------+ +| database | time in minutes: total (insert data/create indexes) | ++======================+======================================================+ +| MySQL 5.0 MyISAM | 205 (160/45) | ++----------------------+------------------------------------------------------+ +| MySQL 5.0 InnoDB | _untested_, see NOTES below | ++----------------------+------------------------------------------------------+ +| PostgreSQL 8.1 | 560 (530/30) | ++----------------------+------------------------------------------------------+ +| SQLite 3.3 | ??? (150/???) -very slow building indexes | +| | | +| | Timed with the "--sqlite-transactions" command | +| | | +| | line option; otherwise it's _really_ slow: | +| | | +| | even 35 hours or more | ++----------------------+------------------------------------------------------+ +| SQLite 3.7 | 65/13 - with --sqlite-transactions | +| | and using an SSD disk | ++----------------------+------------------------------------------------------+ +| SQL Server | about 3 or 4 hours | ++----------------------+------------------------------------------------------+ + +If you have different experiences, please tell me! + +As expected, the most important things that you can do to improve performance +are: + +#. Use an in-memory filesystem or an SSD disk. +#. Use the ``-c /path/to/empty/dir`` argument to use CSV files. +#. Follow the specific notes about your database server. + + +Notes +----- + +[save the output] + +The imdbpy2sql.py will print a lot of debug information on standard output; +you can save it in a file, appending (without quotes) "2>&1 | tee output.txt" + + +[Microsoft Windows paths] + +It's much safer, in a Microsoft Windows environment, to use full paths +for the values of the '-c' and '-d' arguments, complete with drive letter. +The best thing is to use _UNIX_ path separator, and to add a leading +separator, e.g.:: + + -d C:/path/to/imdb_files/ -c C:/path/to/csv_tmp_files/ + + +[MySQL] + +In general, if you get an annoyingly high number of "TOO MANY DATA +... SPLITTING" lines, consider increasing max_allowed_packet +(in the configuration of your MySQL server) to at least 8M or 16M. +Otherwise, inserting the data will be very slow, and some data may +be lost. + + +[MySQL InnoDB and MyISAM] + +InnoDB is abysmal slow for our purposes: my suggestion is to always use +MyISAM tables and -if you really want to use InnoDB- convert the tables +later. The imdbpy2sql.py script provides a simple way to manage these cases, +see ADVANCED FEATURES below. + +In my opinion, the cleaner thing to do is to set the server to use +MyISAM tables or -if you can't modify the server- +use the ``--mysql-force-myisam`` command line option of imdbpy2sql.py. +Anyway, if you really need to use InnoDB, in the server-side settings +I recommend to set innodb_file_per_table to "true". + +Beware that the conversion will be extremely slow (some hours), but still +faster than using InnoDB from the start. You can use the "--mysql-innodb" +command line option to force the creation of a database with MyISAM tables, +converted at the end into InnoDB. + + +[Microsoft SQL Server/SQLExpress] + +If you get and error about how wrong and against nature the blasphemous act +of inserting an identity key is, you can try to fix it with the new custom +queries support; see ADVANCED FEATURES below. + +As a shortcut, you can use the "--ms-sqlserver" command line option +to set all the needed options. + + +[SQLite speed-up] + +For some reason, SQLite is really slow, except when used with transactions; +you can use the "--sqlite-transactions" command line option to obtain +acceptable performance. The same command also turns off "PRAGMA synchronous". + +SQLite seems to hugely benefit from the use of a non-journaling filesystem +and/or of a ramdisk/tmpfs: see the generic suggestions discussed above +in the Timing section. + + +[SQLite failure] + +It seems that with older versions of the python-sqlite package, the first run +may fail; if you get a DatabaseError exception saying "no such table", +try running again the command with the same arguments. Double funny, huh? ;-) + + +[data truncated] + +If you get an insane amount (hundreds or thousands, on various text columns) +of warnings like these: + + imdbpy2sql.py:727: Warning: Data truncated for column 'person_role' at row 4979 + CURS.executemany(self.sqlString, self.converter(self.values())) + +you probably have a problem with the configuration of your database. +The error comes from strings that get cut at the first non-ASCII character +(and so you're losing a lot of information). + +To solves this problem, you must be sure that your database server is set up +properly, with the use library/client configured to communicate with the server +in a consistent way. For example, for MySQL you can set:: + + character-set-server = utf8 + default-collation = utf8_unicode_ci + default-character-set = utf8 + +or even:: + + character-set-server = latin1 + default-collation = latin1_bin + default-character-set = latin1 + + +[adult titles] + +Beware that, while running, the imdbpy2sql.py script will output +a lot of strings containing both person names and movie titles. The script +has absolutely no way of knowing that the processed title is an adult-only +movie, so... if you leave it on and your little daughter runs to you +screaming "daddy! daddy! what kind of animals does Rocco train in the +documentary 'Rocco: Animal Trainer 17'???"... well, it's not my fault! ;-) + + +Advanced features +----------------- + +With the -e (or --execute) command line argument you can specify +custom queries to be executed at certain times, with the syntax:: + + -e "TIME:[OPTIONAL_MODIFIER:]QUERY" + +where TIME is one of: 'BEGIN', 'BEFORE_DROP', 'BEFORE_CREATE', +'AFTER_CREATE', 'BEFORE_MOVIES', 'BEFORE_CAST', 'BEFORE_RESTORE', +'BEFORE_INDEXES', 'END'. + +The only available OPTIONAL_MODIFIER is 'FOR_EVERY_TABLE' and it means +that the QUERY command will be executed for every table in the database +(so it doesn't make much sense to use it with BEGIN, BEFORE_DROP +or BEFORE_CREATE time...), replacing the "%(table)s" text in the QUERY +with the appropriate table name. + +Other available TIMEs are: 'BEFORE_MOVIES_TODB', 'AFTER_MOVIES_TODB', +'BEFORE_PERSONS_TODB', 'AFTER_PERSONS_TODB', 'BEFORE_CHARACTERS_TODB', +'AFTER_CHARACTERS_TODB', 'BEFORE_SQLDATA_TODB', 'AFTER_SQLDATA_TODB', +'BEFORE_AKAMOVIES_TODB' and 'AFTER_AKAMOVIES_TODB'; they take no modifiers. +Special TIMEs 'BEFORE_EVERY_TODB' and 'AFTER_EVERY_TODB' apply to +every BEFORE_* and AFTER_* TIME above mentioned. + +These commands are executed before and after every _toDB() call in +their respective objects (CACHE_MID, CACHE_PID and SQLData instances); +the "%(table)s" text in the QUERY is replaced as above. + +You can specify so many -e arguments as you need, even if they refer +to the same TIME: they will be executed from the first to the last. +Also, always remember to correctly escape queries: after all you're +passing it on the command line! + +E.g. (ok, quite a silly example...):: + + -e "AFTER_CREATE:SELECT * FROM title;" + +The most useful case is when you want to convert the tables of a MySQL +from MyISAM to InnoDB:: + + -e "END:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;" + +If your system uses InnoDB by default, you can trick it with:: + + -e "AFTER_CREATE:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=MyISAM;" -e "END:FOR_EVERY_TABLE:ALTER TABLE %(table)s ENGINE=InnoDB;" + +You can use the "--mysql-innodb" command line option as a shortcut +of the above command. + +Cool, huh? + +Another possible use is to fix a problem with Microsoft SQLServer/SQLExpress. +To prevent errors setting IDENTITY fields, you can run something like this:: + + -e 'BEFORE_EVERY_TODB:SET IDENTITY_INSERT %(table)s ON' -e 'AFTER_EVERY_TODB:SET IDENTITY_INSERT %(table)s OFF' + +You can use the "--ms-sqlserver" command line option as a shortcut +of the above command. + +To use transactions to speed-up SQLite, try:: + + -e 'BEFORE_EVERY_TODB:BEGIN TRANSACTION;' -e 'AFTER_EVERY_TODB:COMMIT;' + +Which is also the same thing the command line option "--sqlite-transactions" +does. + + +CSV files +--------- + +.. note:: + + Keep in mind that not all database servers support this. + + Moreover, you can run into problems. For example, if you're using + PostgreSQL, your server process will need read access to the directory + where the CSV files are stored. + +To create the database using a set of CSV files, run :file:`imdbpy2sql.py` +as follows:: + + imdbpy2sql.py -d /dir/with/plainTextDataFiles/ -u URI \ + -c /path/to/the/csv_files_dir/ + +The created CSV files will be imported near the end of processing. After the import +is finished, you can safely remove these files. + +Since version 4.5, it's possible to separate the two steps involved +when using CSV files: + +- With the ``--csv-only-write`` command line option, the old database will be + truncated and the CSV files saved, along with imdbID information. + +- With the ``--csv-only-load`` option, these saved files can be loaded + into an existing database (this database MUST be the one left almost empty + by the previous run). + +Beware that right now the whole procedure is not very well tested. +For both commands, you still have to specify the whole +``-u URI -d /path/plainTextDataFiles/ -c /path/CSVfiles/`` arguments. + + +.. _SQLAlchemy: https://www.sqlalchemy.org/ +.. _SQLAlchemy dialects: http://docs.sqlalchemy.org/en/latest/dialects/ +.. _this comment by Andrew D Bate: https://github.com/alberanid/imdbpy/issues/130#issuecomment-365707620 diff -pruN 5.1-1/docs/usage/query.rst 6.6-1/docs/usage/query.rst --- 5.1-1/docs/usage/query.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/query.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,105 @@ +:orphan: + +Querying data +============= + +Method descriptions: + +``search_movie(title)`` + Searches for the given title, and returns a list of Movie objects containing + only basic information like the movie title and year, and with a "movieID" + instance variable: + + - ``movieID`` is an identifier of some kind; for the sake of simplicity + you can think of it as the ID used by the IMDb's web server used + to uniquely identify a movie (e.g.: '0094226' for Brian De Palma's + "The Untouchables"), but keep in mind that it's not necessary the + same ID!!! + + For some implementations of the "data access system" these two IDs can + be the same (as is the case for the 'http' data access system), but + other access systems can use a totally different kind of movieID. + The easier (I hope!) way to understand this is to think of the movieID + returned by the search_movie() method as the *thing* you have to pass + to the get_movie() method, so that it can retrieve info about the referred + movie. + + So, movieID *can* be the imdbID ('0094226') if you're accessing the web + server, but with a SQL installation of the IMDb database, movieID will be + an integer, as read from the id column in the database. + +``search_episode(title)`` + This is identical to ``search_movie()``, except that it is tailored + to searching for titles of TV series episodes. Best results are expected + when searching for just the title of the episode, *without* the title + of the TV series. + +``get_movie(movieID)`` + This will fetch the needed data and return a Movie object for the movie + referenced by the given movieID. The Movie class can be found in the Movie + module. A Movie object presents basically the same interface of a Python's + dictionary; so you can access, for example, the list of actors and actresses + using the syntax ``movieObject['cast']``. + +The ``search_person(name)``, ``get_person(personID)``, +``search_character(name)``, ``get_character(characterID)``, +``search_company(name)``, and ``get_company(companyID)`` methods work the same +way as ``search_movie(title)`` and ``get_movie(movieID)``. + +The ``search_keyword(string)`` method returns a list of strings that are +valid keywords, similar to the one given. + +The ``get_keyword(keyword)`` method returns a list of Movie instances that +are tagged with the given keyword. + +The ``get_imdbMovieID(movieID)``, ``get_imdbPersonID(personID)``, +``get_imdbCharacterID(characterID)``, and ``get_imdbCompanyID(companyID)`` +methods take, respectively, a movieID, a personID, a movieID, or a companyID +and return the relative imdbID; it's safer to use the +``get_imdbID(MovieOrPersonOrCharacterOrCompanyObject)`` method. + +The ``title2imdbID(title)``, ``name2imdbID(name)``, ``character2imdbID(name)``, +and ``company2imdbID(name)`` methods take, respectively, a movie title +(in the plain text data files format), a person name, a character name, or +a company name, and return the relative imdbID; when possible it's safer +to use the ``get_imdbID(MovieOrPersonOrCharacterOrCompanyObject)`` method. + +The ``get_imdbID(MovieOrPersonOrCharacterOrCompanyObject)`` method returns +the imdbID for the given Movie, Person, Character or Company object. + +The ``get_imdbURL(MovieOrPersonOrCharacterOrCompanyObject)`` method returns +a string with the main IMDb URL for the given Movie, Person, Character, or +Company object; it does its best to retrieve the URL. + +The ``update(MovieOrPersonOrCharacterOrCompanyObject)`` method takes +an instance of a Movie, Person, Character, or Company class, and retrieves +other available information. + +Remember that the ``search_*(txt)`` methods will return a list of Movie, +Person, Character or Company objects with only basic information, +such as the movie title or the person/character name. So, ``update()`` can be +used to retrieve every other information. + +By default a "reasonable" set of information are retrieved: 'main', +'filmography', and 'biography' for a Person/Character object; 'main' and 'plot' +for a Movie object; 'main' for a Company object. + +Example: + +.. code-block:: python + + # only basic information like the title will be printed. + print(first_match.summary()) + # update the information for this movie. + i.update(first_match) + # a lot of information will be printed! + print(first_match.summary()) + # retrieve trivia information + i.update(first_match, 'trivia') + print(m['trivia']) + # retrieve both 'quotes' and 'goofs' information (with a list or tuple) + i.update(m, ['quotes', 'goofs']) + print(m['quotes']) + print(m['goofs']) + # retrieve every available information. + i.update(m, 'all') diff -pruN 5.1-1/docs/usage/quickstart.rst 6.6-1/docs/usage/quickstart.rst --- 5.1-1/docs/usage/quickstart.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/quickstart.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,160 @@ +Quick start +=========== + +The first thing to do is to import :mod:`imdb` and call the :mod:`imdb.IMDb` +function to get an access object through which IMDb data can be retrieved: + +.. code-block:: python + + >>> import imdb + >>> ia = imdb.IMDb() + +By default this will fetch the data from the IMDb web server but there are +other options. See the :ref:`access systems ` document +for more information. + +Searching +--------- + +You can use the :meth:`search_movie ` method +of the access object to search for movies with a given (or similar) title. +For example, to search for movies with titles like "matrix": + +.. code-block:: python + + >>> movies = ia.search_movie('matrix') + >>> movies[0] + + +Similarly, you can search for people and companies using +the :meth:`search_person ` and +the :meth:`search_company ` methods: + +.. code-block:: python + + >>> people = ia.search_person('angelina') + >>> people[0] + + >>> companies = ia.search_company('rko') + >>> companies[0] + + +As the examples indicate, the results are lists of +:class:`Movie `, :class:`Person `, or +:class:`Company ` objects. These behave like +dictionaries, i.e. they can be queried by giving the key of the data +you want to obtain: + +.. code-block:: python + + >>> movies[0]['title'] + 'The Matrix' + >>> people[0]['name'] + 'Angelina Jolie' + >>> companies[0]['name'] + 'RKO' + +Movie, person, and company objects have id attributes which +-when fetched through the IMDb web server- store the IMDb id +of the object: + +.. code-block:: python + + >>> movies[0].movieID + '0133093' + >>> people[0].personID + '0001401' + >>> companies[0].companyID + '0226417' + + + +Retrieving +---------- + +If you know the IMDb id of a movie, you can use +the :meth:`get_movie ` method to retrieve its data. +For example, the movie "The Untouchables" by Brian De Palma has the id +"0094226": + +.. code-block:: python + + >>> movie = ia.get_movie('0094226') + >>> movie + + +Similarly, the :meth:`get_person ` and +the :meth:`get_company ` methods can be used +for retrieving :class:`Person ` and +:class:`Company ` data: + +.. code-block:: python + + >>> person = ia.get_person('0000206') + >>> person['name'] + 'Keanu Reeves' + >>> person['birth date'] + '1964-9-2' + >>> company = ia.get_company('0017902') + >>> company['name'] + 'Pixar Animation Studios' + + +Keywords +-------- + +You can search for keywords similar to the one provided: + +.. code-block:: python + + >>> keywords = ia.search_keyword('dystopia') + >>> keywords + ['dystopia', 'dystopian-future', ..., 'dystopic-future'] + +And movies that match a given keyword: + +.. code-block:: python + + >>> movies = ia.get_keyword('dystopia') + >>> len(movies) + 50 + >>> movies[0] + + + +Top / bottom movies +------------------- + +It's possible to retrieve the list of top 250 and bottom 100 movies: [#sql_bottom]_ + +.. code-block:: python + + >>> top = ia.get_top250_movies() + >>> top[0] + + >>> bottom = ia.get_bottom100_movies() + >>> bottom[0] + + + +Exceptions +---------- + +Any error related to IMDbPY can be caught by checking for +the :class:`imdb.IMDbError` exception: + +.. code-block:: python + + from imdb import IMDb, IMDbError + + try: + ia = IMDb() + people = ia.search_person('Mel Gibson') + except IMDbError as e: + print(e) + + +.. [#sql_bottom] + + Beware that in an SQL-based access system, the bottom 100 list is limited + to the first 10 results. diff -pruN 5.1-1/docs/usage/role.rst 6.6-1/docs/usage/role.rst --- 5.1-1/docs/usage/role.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/role.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,191 @@ +Roles +===== + +When parsing data of a movie, you'll encounter references to the people +who worked on it, like its cast, director and crew members. + +For people in the cast (actors and actresses), +the :attr:`currentRole ` attribute is set to the name +of the character they played: + +.. code-block:: python + + >>> movie = ia.get_movie('0075860') + >>> movie + + >>> actor = movie['cast'][6] + >>> actor + + >>> actor['name'] + 'Warren J. Kemmerling' + >>> actor.currentRole + 'Wild Bill' + +Miscellaneous data, such as an AKA name for the actor or an "uncredited" +notice, is stored in the :attr:`notes ` attribute: + +.. code-block:: python + + >>> actor.notes + '(as Warren Kemmerling)' + +For crew members other than the cast, +the :attr:`notes ` attribute contains the description +of the person's job: + +.. code-block:: python + + >>> crew_member = movie['art department'][0] + >>> crew_member + + >>> crew_member.notes + 'property master' + +The ``in`` operator can be used to check whether a person worked in a given +movie or not: + +.. code-block:: python + + >>> movie + + >>> actor + + >>> actor in movie + True + >>> crew_member + + >>> crew_member in movie + True + >>> person + + >>> person in movie + False + +Obviously these Person objects contain only information directly +available upon parsing the movie pages, e.g.: the name, an imdbID, the role. +So if now you write:: + + print(writer['actor']) + +to get a list of movies acted by Mel Gibson, you'll get a KeyError exception, +because the Person object doesn't contain this kind of information. + + +The same is true when parsing person data: you'll find a list of movie +the person worked on and, for every movie, the currentRole instance variable +is set to a string describing the role of the considered person: + +.. code-block:: python + + # Julia Roberts + julia = i.get_person('0000210') + # Output a list of movies she acted in and the played role + # separated by '::' + print([movie['title'] + '::' + movie.currentRole + for movie in julia['actress']]) + +Here the various Movie objects only contain minimal information, +like the title and the year; the latest movie with Julia Roberts: + +.. code-block:: python + + last = julia['actress'][0] + # Retrieve full information + i.update(last) + # name of the first director + print(last['director'][0]['name']) + + +.. note:: + + Since the end of 2017, IMDb has removed the Character kind of information. + This document is still valid, but only for the obsolete "sql" data access + system. + +Since version 3.3, IMDbPY supports the character pages of the IMDb database; +this required some substantial changes to how actors' and acresses' roles +were handled. Starting with release 3.4, "sql" data access system is supported, +too - but it works a bit differently from "http". See "SQL" below. + +The currentRole instance attribute can be found in every instance of Person, +Movie and Character classes, even if actually the Character never uses it. + +The currentRole of a Person object is set to a Character instance, inside +a list of person who acted in a given movie. The currentRole of a Movie object +is set to a Character instance, inside a list of movies played be given person. +The currentRole of a Movie object is set to a Person instance, inside a list +of movies in which a given character was portrayed. + +Schema:: + + movie['cast'][0].currentRole -> a Character object. + | + +-> a Person object. + + person['actor'][0].currentRole -> a Character object. + | + +-> a Movie object. + + character['filmography'][0].currentRole -> a Person object. + | + +-> a Movie object. + +The roleID attribute can be used to access/set the characterID or personID +instance attribute of the current currentRole. When building Movie or Person +objects, you can pass the currentRole parameter and the roleID parameter +(to set the ID). The currentRole parameter can be an object +(Character or Person), a string (in which case a Character or Person object is +automatically instantiated) or a list of objects or strings (to handle multiple +characters played by the same actor/actress in a movie, or character played +by more then a single actor/actress in the same movie). + +Anyway, currentRole objects (Character or Person instances) can be +pretty-printed easily: calling unicode(CharacterOrPersonObject) will return +a good-old-string. + + +SQL +--- + +Fetching data from the web, only characters with an active page on the web site +will have their characterID; we don't have these information when accessing +through "sql", so *every* character will have an associated characterID. +This way, every character with the same name will share the same characterID, +even if - in fact - they may not be portraying the same character. + + +Goodies +------- + +To help getting the required information from Movie, Person and Character +objects, in the "helpers" module there's a new factory function, +makeObject2Txt, which can be used to create your pretty-printing function. +It takes some optional parameters: movieTxt, personTxt, characterTxt +and companyTxt; in these strings %(value)s items are replaced with +object['value'] or with obj.value (if the first is not present). + +E.g.: + +.. code-block:: python + + import imdb + myPrint = imdb.helpers.makeObject2Txt(personTxt=u'%(name)s ... %(currentRole)s') + i = imdb.IMDb() + m = i.get_movie('0057012') + ps = m['cast'][0] + print(myPrint(ps)) + # The output will be something like: + # Peter Sellers ... Group Captain Lionel Mandrake / President Merkin Muffley / Dr. Strangelove + + +Portions of the formatting string can be stripped conditionally: +if the specified condition is false, they will be cancelled. + +E.g.:: + + myPrint = imdb.helpers.makeObject2Txt(personTxt='%(long imdb name)s ... %(currentRole)s %(notes)s' + + +Another useful argument is 'applyToValues': if set to a function, it will be +applied to every value before the substitution; it can be useful to format +strings for HTML output. diff -pruN 5.1-1/docs/usage/s3.rst 6.6-1/docs/usage/s3.rst --- 5.1-1/docs/usage/s3.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/s3.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,65 @@ +.. _s3: + +S3 datasets +=========== + +IMDb distributes some of its data as downloadable `datasets`_. IMDbPY can +import this data into a database and make it accessible through its API. +[#ptdf]_ + +For this, you will first need to install `SQLAlchemy`_ and the libraries +that are needed for the database server you want to use. Check out +the `SQLAlchemy dialects`_ documentation for more detail. + +Then, follow these steps: + +#. Download the files from the following address and put all of them + in the same directory: https://datasets.imdbws.com/ + +#. Create a database. Use a collation like ``utf8_unicode_ci``. + +#. Import the data using the :file:`s32imdbpy.py` script:: + + s32imdbpy.py /path/to/the/tsv.gz/files/ URI + + *URI* is the identifier used to access the SQL database. For example:: + + s32imdbpy.py ~/Download/imdb-s3-dataset-2018-02-07/ \ + postgres://user:password@localhost/imdb + +Once the import is finished -which should take about an hour or less +on a modern system- you will have a SQL database with all the information +and you can use the normal IMDbPY API: + +.. code-block:: python + + from imdb import IMDb + + ia = IMDb('s3', 'postgres://user:password@localhost/imdb') + + results = ia.search_movie('the matrix') + for result in results: + print(result.movieID, result) + + matrix = results[0] + ia.update(matrix) + print(matrix.keys()) + + +.. note:: + + Running the script again will drop the current tables and import + the data again. + + +.. [#ptdf] + + Until the end of 2017, IMDb used to distribute a more comprehensive subset + of its data in a different format. IMDbPY can also import that data + but note that the data is not being updated anymore. For more information, + see :ref:`ptdf`. + + +.. _datasets: https://www.imdb.com/interfaces/ +.. _SQLAlchemy: https://www.sqlalchemy.org/ +.. _SQLAlchemy dialects: http://docs.sqlalchemy.org/en/latest/dialects/ diff -pruN 5.1-1/docs/usage/series.rst 6.6-1/docs/usage/series.rst --- 5.1-1/docs/usage/series.rst 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/docs/usage/series.rst 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,214 @@ +Series +====== + +As on the IMDb site, each TV series and also each of a TV series' episodes is +treated as a regular title, just like a movie. The ``kind`` key can be used +to distinguish series and episodes from movies: + +.. code-block:: python + + >>> series = ia.get_movie('0389564') + >>> series + + >>> series['kind'] + 'tv series' + >>> episode = ia.get_movie('0502803') + >>> episode + + >>> episode['kind'] + 'episode' + +The episodes of a series can be fetched using the "episodes" infoset. This +infoset adds an ``episodes`` key which is a dictionary from season numbers +to episodes. And each season is a dictionary from episode numbers within +the season to the episodes. Note that the season and episode numbers don't +start from 0; they are the numbers given by the IMDb: + +.. code-block:: python + + >>> ia.update(series, 'episodes') + >>> sorted(series['episodes'].keys()) + [1, 2, 3, 4] + >>> season4 = series['episodes'][4] + >>> len(season4) + 13 + >>> episode = series['episodes'][4][2] + >>> episode + + >>> episode['season'] + 4 + >>> episode['episode'] + 2 + +The title of the episode doesn't contain the title of the series: + +.. code-block:: python + + >>> episode['title'] + 'Fear Itself' + >>> episode['series title'] + 'The 4400' + +The episode also contains a key that refers to the series, but beware that, +to avoid circular references, it's not the same object as the series object +we started with: + +.. code-block:: python + + >>> episode['episode of'] + + >>> series + + + +Titles +------ + +The ``analyze_title()`` and ``build_title()`` functions now support +TV episodes. You can pass a string to the ``analyze_title`` function +in the format used by the web server (``"The Series" The Episode (2005)``) +or in the format of the plain text data files +(``"The Series" (2004) {The Episode (#ser.epi)}``). + +For example, if you call the function:: + + analyze_title('"The Series" The Episode (2005)') + +the result will be:: + + { + 'kind': 'episode', # kind is set to 'episode' + 'year': '2005', # release year of this episode + 'title': 'The Episode', # episode title + 'episode of': { # 'episode of' will contain + 'kind': 'tv series', # information about the series + 'title': 'The Series' + } + } + + +The ``episode of`` key can be a dictionary or a ``Movie`` instance +with the same information. + +The ``build_title()`` function takes an optional argument: ``ptdf``, +which when set to false (the default) returns the title of the episode +in the format used by the IMDb's web server +("The Series" An Episode (2006)); otherwise, it uses the format used +by the plain text data files (something like +"The Series" (2004) {An Episode (#2.5)}) + + +Full credits +------------ + +When retrieving credits for a TV series or mini-series, you may notice that +many long lists (like "cast" and "writers") are incomplete. You can fetch +the complete list of cast and crew with the "full credits" data set: + +.. code-block:: python + + >>> series = ia.get_movie('0285331') + >>> series + + >>> len(series['cast']) + 50 + >>> ia.update(series, 'full credits') + >>> len(series['cast']) + 2514 + +If you prefer, you can retrieve the complete cast of every episode, keeping +the lists separated for each episode. Instead of retrieving with:: + + ia.update(series, 'episodes') + +use:: + + ia.update(series, 'episodes cast') + +or the equivalent:: + + i.update(m, 'guests') + +Now you end up having the same information as if you have updated +the 'episodes' info set, but every Movie object inside the dictionary +of dictionary has the complete cast, e.g.:: + + cast = m['episodes'][1][2]['cast'] # cast list for the second episode + # of the first season. + +Beware that both 'episodes cast' and 'guests' will update the +keyword 'episodes' (and not 'episodes cast' or 'guests'). + + +Ratings +------- + +You can retrieve rating information about every episode in a TV series +or mini series using the 'episodes rating' data set. + + +People +------ + +You can retrieve information about single episodes acted/directed/... +by a person. + +.. code-block:: python + + from imdb import IMDb + i = IMDb() + p = i.get_person('0005041') # Laura Innes + p['actress'][0] # + + # At this point you have an entry (in keys like 'actor', 'actress', + # 'director', ...) for every series the person starred/worked in, but + # you knows nothing about singles episodes. + i.update(p, 'episodes') # updates information about single episodes. + + p['episodes'] # a dictionary with the format: + # {: [ + # , + # , + # ... + # ], + # ... + # } + + er = p['actress'][0] # ER tv series + p['episodes'][er] # list of Movie objects; one for every ER episode + # she starred/worked in + + p['episodes'][er][0] # + p['episodes'][er]['kind'] # 'episode' + p['episodes'][er][0].currentRole # 'Dr. Kerry Weaver' + + +Goodies +------- + +In the ``imdb.helpers`` module there are some functions useful to manage +lists of episodes: + +- ``sortedSeasons(m)`` returns a sorted list of seasons of the given series, e.g.: + + .. code-block:: python + + >>> from imdb import IMDb + >>> i = IMDb() + >>> m = i.get_movie('0411008') + >>> i.update(m, 'episodes') + >>> sortedSeasons(m) + [1, 2] + +- ``sortedEpisodes(m, season=None)`` returns a sorted list of episodes of the + the given series for only the specified season(s) (if None, every season), + e.g.: + + .. code-block:: python + + >>> from imdb import IMDb + >>> i = IMDb() + >>> m = i.get_movie('0411008') + >>> i.update(m, 'episodes') + >>> sortedEpisodes(m, season=1) + [, , ...] diff -pruN 5.1-1/ez_setup.py 6.6-1/ez_setup.py --- 5.1-1/ez_setup.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/ez_setup.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,284 +0,0 @@ -#!python -"""Bootstrap setuptools installation - -If you want to use setuptools in your package's setup.py, just include this -file in the same directory with it, and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -If you want to require a specific version of setuptools, set a download -mirror, or use an alternate download directory, you can do so by supplying -the appropriate options to ``use_setuptools()``. - -This file can also be run as a script to install or upgrade setuptools. -""" -import sys -DEFAULT_VERSION = "0.6c11" -DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] - -md5_data = { - 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca', - 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb', - 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b', - 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a', - 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618', - 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac', - 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5', - 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4', - 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c', - 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b', - 'setuptools-0.6c10-py2.3.egg': 'ce1e2ab5d3a0256456d9fc13800a7090', - 'setuptools-0.6c10-py2.4.egg': '57d6d9d6e9b80772c59a53a8433a5dd4', - 'setuptools-0.6c10-py2.5.egg': 'de46ac8b1c97c895572e5e8596aeb8c7', - 'setuptools-0.6c10-py2.6.egg': '58ea40aef06da02ce641495523a0b7f5', - 'setuptools-0.6c11-py2.3.egg': '2baeac6e13d414a9d28e7ba5b5a596de', - 'setuptools-0.6c11-py2.4.egg': 'bd639f9b0eac4c42497034dec2ec0c2b', - 'setuptools-0.6c11-py2.5.egg': '64c94f3bf7a72a13ec83e0b24f2749b2', - 'setuptools-0.6c11-py2.6.egg': 'bfa92100bd772d5a213eedd356d64086', - 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27', - 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277', - 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa', - 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e', - 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e', - 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f', - 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2', - 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc', - 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167', - 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64', - 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d', - 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20', - 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab', - 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53', - 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2', - 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e', - 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372', - 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902', - 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de', - 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b', - 'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03', - 'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a', - 'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6', - 'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a', -} - -import sys, os -try: from hashlib import md5 -except ImportError: from md5 import md5 - -def _validate_md5(egg_name, data): - if egg_name in md5_data: - digest = md5(data).hexdigest() - if digest != md5_data[egg_name]: - print >>sys.stderr, ( - "md5 validation of %s failed! (Possible download problem?)" - % egg_name - ) - sys.exit(2) - return data - -def use_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, - download_delay=15 -): - """Automatically find/download setuptools and make it available on sys.path - - `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end with - a '/'). `to_dir` is the directory where setuptools will be downloaded, if - it is not already available. If `download_delay` is specified, it should - be the number of seconds that will be paused before initiating a download, - should one be required. If an older version of setuptools is installed, - this routine will print a message to ``sys.stderr`` and raise SystemExit in - an attempt to abort the calling script. - """ - was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules - def do_download(): - egg = download_setuptools(version, download_base, to_dir, download_delay) - sys.path.insert(0, egg) - import setuptools; setuptools.bootstrap_install_from = egg - try: - import pkg_resources - except ImportError: - return do_download() - try: - pkg_resources.require("setuptools>="+version); return - except pkg_resources.VersionConflict, e: - if was_imported: - print >>sys.stderr, ( - "The required version of setuptools (>=%s) is not available, and\n" - "can't be installed while this script is running. Please install\n" - " a more recent version first, using 'easy_install -U setuptools'." - "\n\n(Currently using %r)" - ) % (version, e.args[0]) - sys.exit(2) - else: - del pkg_resources, sys.modules['pkg_resources'] # reload ok - return do_download() - except pkg_resources.DistributionNotFound: - return do_download() - -def download_setuptools( - version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, - delay = 15 -): - """Download setuptools from a specified location and return its filename - - `version` should be a valid setuptools version number that is available - as an egg for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download attempt. - """ - import urllib2, shutil - egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) - url = download_base + egg_name - saveto = os.path.join(to_dir, egg_name) - src = dst = None - if not os.path.exists(saveto): # Avoid repeated downloads - try: - from distutils import log - if delay: - log.warn(""" ---------------------------------------------------------------------------- -This script requires setuptools version %s to run (even to display -help). I will attempt to download it for you (from -%s), but -you may need to enable firewall access for this script first. -I will start the download in %d seconds. - -(Note: if this machine does not have network access, please obtain the file - - %s - -and place it in this directory before rerunning this script.) ----------------------------------------------------------------------------""", - version, download_base, delay, url - ); from time import sleep; sleep(delay) - log.warn("Downloading %s", url) - src = urllib2.urlopen(url) - # Read/write all in one block, so we don't create a corrupt file - # if the download is interrupted. - data = _validate_md5(egg_name, src.read()) - dst = open(saveto,"wb"); dst.write(data) - finally: - if src: src.close() - if dst: dst.close() - return os.path.realpath(saveto) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -def main(argv, version=DEFAULT_VERSION): - """Install or upgrade setuptools and EasyInstall""" - try: - import setuptools - except ImportError: - egg = None - try: - egg = download_setuptools(version, delay=0) - sys.path.insert(0,egg) - from setuptools.command.easy_install import main - return main(list(argv)+[egg]) # we're done here - finally: - if egg and os.path.exists(egg): - os.unlink(egg) - else: - if setuptools.__version__ == '0.0.1': - print >>sys.stderr, ( - "You have an obsolete version of setuptools installed. Please\n" - "remove it from your system entirely before rerunning this script." - ) - sys.exit(2) - - req = "setuptools>="+version - import pkg_resources - try: - pkg_resources.require(req) - except pkg_resources.VersionConflict: - try: - from setuptools.command.easy_install import main - except ImportError: - from easy_install import main - main(list(argv)+[download_setuptools(delay=0)]) - sys.exit(0) # try to force an exit - else: - if argv: - from setuptools.command.easy_install import main - main(argv) - else: - print "Setuptools version",version,"or greater has been installed." - print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' - -def update_md5(filenames): - """Update our built-in md5 registry""" - - import re - - for name in filenames: - base = os.path.basename(name) - f = open(name,'rb') - md5_data[base] = md5(f.read()).hexdigest() - f.close() - - data = [" %r: %r,\n" % it for it in md5_data.items()] - data.sort() - repl = "".join(data) - - import inspect - srcfile = inspect.getsourcefile(sys.modules[__name__]) - f = open(srcfile, 'rb'); src = f.read(); f.close() - - match = re.search("\nmd5_data = {\n([^}]+)}", src) - if not match: - print >>sys.stderr, "Internal error!" - sys.exit(2) - - src = src[:match.start(1)] + repl + src[match.end(1):] - f = open(srcfile,'w') - f.write(src) - f.close() - - -if __name__=='__main__': - if len(sys.argv)>2 and sys.argv[1]=='--md5update': - update_md5(sys.argv[2:]) - else: - main(sys.argv[1:]) - - - - - - diff -pruN 5.1-1/.github/ISSUE_TEMPLATE.md 6.6-1/.github/ISSUE_TEMPLATE.md --- 5.1-1/.github/ISSUE_TEMPLATE.md 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/.github/ISSUE_TEMPLATE.md 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,36 @@ +#### Issue description + +*write the description here* + + +#### Version of IMDbPY, Python and OS + +- **Python:** `python3 -V` or, if you are using Python 2, `python -V` +- **IMDbPY:** `python3 -c 'import imdb ; print(imdb.VERSION)'` or, if you are using Python 2, `python -c 'import imdb ; print(imdb.VERSION)'` +- **OS:** `python -c 'import platform ; print(platform.uname())'` + + +#### Steps to reproduce the issue + +*if possible, provide a minimal code to reproduce the problem* + +``` +#!python + +# your code here +``` + + +#### What's the expected result? + +- + + +#### What's the actual result? + +- + + +#### Additional details + +- diff -pruN 5.1-1/.gitignore 6.6-1/.gitignore --- 5.1-1/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/.gitignore 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,16 @@ +*.pyc +*.pyo +*.egg-info +*.mo +*.so +.pytest_cache/ +build/ +_build/ +__pycache__ +dist/ +.idea +.vscode +.cache +.tox +.coverage +prof diff -pruN 5.1-1/.hgignore 6.6-1/.hgignore --- 5.1-1/.hgignore 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/.hgignore 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,18 @@ +syntax: glob +.cache +.tox +__pycache__ +build +dist +.cache +.tox +__pycache__ +*.egg-info +*.mo +*.pyc +*.pyo +*.so +*.pyd +*~ +*.swp +setuptools-*.egg diff -pruN 5.1-1/.hgtags 6.6-1/.hgtags --- 5.1-1/.hgtags 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/.hgtags 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,8 @@ +c3dba80881f0a810b3bf93051a56190b297e7a50 4.6 +c8b07121469a2173a587b1a34beb4f1fecd640b6 4.7 +ba221c9050599463b4b78c89a8bdada7d7aef173 4.8 +e807ba790392d406018af0f98d5dad5117721a4d 4.8.1 +b02c61369b27e0d5af0a755a8a2fc3355c08bb67 4.8.2 +7f39f8ac4838b45fbf59f4167796dd17cd15c437 4.9 +398c01b961076362958c27584d85fbdfa921ac63 5.0 +cb1e19b508d03499e8f34bd066d8b930aca6aa2d 5.1 diff -pruN 5.1-1/imdb/Character.py 6.6-1/imdb/Character.py --- 5.1-1/imdb/Character.py 2015-08-28 15:14:32.000000000 +0000 +++ 6.6-1/imdb/Character.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,36 +1,39 @@ -""" -Character module (imdb package). +# Copyright 2007-2018 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides the Character class, used to store information about a given character. - -Copyright 2007-2010 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +from __future__ import absolute_import, division, print_function, unicode_literals + from copy import deepcopy -from imdb.utils import analyze_name, build_name, flatten, _Container, cmpPeople +from imdb._exceptions import IMDbParserError +from imdb.utils import _Container, analyze_name, build_name, cmpPeople, flatten class Character(_Container): """A Character. - Every information about a character can be accessed as: + Every information about a character can be accessed as:: + characterObject['information'] + to get a list of the kind of information stored in a Character object, use the keys() method; some useful aliases are defined (as "also known as" for the "akas" key); @@ -40,18 +43,20 @@ class Character(_Container): default_info = ('main', 'filmography', 'biography') # Aliases for some not-so-intuitive keys. - keys_alias = {'mini biography': 'biography', - 'bio': 'biography', - 'character biography': 'biography', - 'character biographies': 'biography', - 'biographies': 'biography', - 'character bio': 'biography', - 'aka': 'akas', - 'also known as': 'akas', - 'alternate names': 'akas', - 'personal quotes': 'quotes', - 'keys': 'keywords', - 'keyword': 'keywords'} + keys_alias = { + 'mini biography': 'biography', + 'bio': 'biography', + 'character biography': 'biography', + 'character biographies': 'biography', + 'biographies': 'biography', + 'character bio': 'biography', + 'aka': 'akas', + 'also known as': 'akas', + 'alternate names': 'akas', + 'personal quotes': 'quotes', + 'keys': 'keywords', + 'keyword': 'keywords' + } keys_tomodify_list = ('biography', 'quotes') @@ -73,129 +78,115 @@ class Character(_Container): *modFunct* -- function called returning text fields. """ name = kwds.get('name') - if name and not self.data.has_key('name'): + if name and 'name' not in self.data: self.set_name(name) self.characterID = kwds.get('characterID', None) - self.myName = kwds.get('myName', u'') + self.myName = kwds.get('myName', '') def _reset(self): """Reset the Character object.""" self.characterID = None - self.myName = u'' + self.myName = '' def set_name(self, name): """Set the name of the character.""" - # XXX: convert name to unicode, if it's a plain string? try: - d = analyze_name(name, canonical=0) + d = analyze_name(name, canonical=False) self.data.update(d) - except: - # TODO: catch only IMDbPYParserError and issue a warning. + except IMDbParserError: pass def _additional_keys(self): """Valid keys to append to the data.keys() list.""" addkeys = [] - if self.data.has_key('name'): + if 'name' in self.data: addkeys += ['long imdb name'] - if self.data.has_key('headshot'): + if 'headshot' in self.data: addkeys += ['full-size headshot'] return addkeys def _getitem(self, key): """Handle special keys.""" - ## XXX: can a character have an imdbIndex? - if self.data.has_key('name'): + # XXX: can a character have an imdbIndex? + if 'name' in self.data: if key == 'long imdb name': return build_name(self.data) - if key == 'full-size headshot' and self.data.has_key('headshot'): - return self._re_fullsizeURL.sub('', self.data.get('headshot', '')) return None def getID(self): """Return the characterID.""" return self.characterID - def __nonzero__(self): + def __bool__(self): """The Character is "false" if the self.data does not contain a name.""" # XXX: check the name and the characterID? - if self.data.get('name'): return 1 - return 0 + return bool(self.data.get('name')) def __contains__(self, item): """Return true if this Character was portrayed in the given Movie or it was impersonated by the given Person.""" - from Movie import Movie - from Person import Person + from .Movie import Movie + from .Person import Person if isinstance(item, Person): - for m in flatten(self.data, yieldDictKeys=1, scalar=Movie): + for m in flatten(self.data, yieldDictKeys=True, scalar=Movie): if item.isSame(m.currentRole): - return 1 + return True elif isinstance(item, Movie): - for m in flatten(self.data, yieldDictKeys=1, scalar=Movie): + for m in flatten(self.data, yieldDictKeys=True, scalar=Movie): if item.isSame(m): - return 1 - return 0 + return True + elif isinstance(item, str): + return item in self.data + return False def isSameName(self, other): """Return true if two character have the same name and/or characterID.""" if not isinstance(other, self.__class__): - return 0 - if self.data.has_key('name') and \ - other.data.has_key('name') and \ - build_name(self.data, canonical=0) == \ - build_name(other.data, canonical=0): - return 1 + return False + if 'name' in self.data and 'name' in other.data and \ + build_name(self.data, canonical=False) == build_name(other.data, canonical=False): + return True if self.accessSystem == other.accessSystem and \ self.characterID is not None and \ self.characterID == other.characterID: - return 1 - return 0 + return True + return False isSameCharacter = isSameName def __deepcopy__(self, memo): """Return a deep copy of a Character instance.""" - c = Character(name=u'', characterID=self.characterID, - myName=self.myName, myID=self.myID, - data=deepcopy(self.data, memo), - notes=self.notes, accessSystem=self.accessSystem, - titlesRefs=deepcopy(self.titlesRefs, memo), - namesRefs=deepcopy(self.namesRefs, memo), - charactersRefs=deepcopy(self.charactersRefs, memo)) + c = Character(name='', characterID=self.characterID, + myName=self.myName, myID=self.myID, + data=deepcopy(self.data, memo), + notes=self.notes, accessSystem=self.accessSystem, + titlesRefs=deepcopy(self.titlesRefs, memo), + namesRefs=deepcopy(self.namesRefs, memo), + charactersRefs=deepcopy(self.charactersRefs, memo)) c.current_info = list(self.current_info) c.set_mod_funct(self.modFunct) return c def __repr__(self): """String representation of a Character object.""" - r = '' % (self.characterID, - self.accessSystem, - self.get('name')) - if isinstance(r, unicode): r = r.encode('utf_8', 'replace') - return r + return '' % ( + self.characterID, self.accessSystem, self.get('name') + ) def __str__(self): """Simply print the short name.""" - return self.get('name', u'').encode('utf_8', 'replace') - - def __unicode__(self): - """Simply print the short title.""" - return self.get('name', u'') + return self.get('name', '') def summary(self): """Return a string with a pretty-printed summary for the character.""" - if not self: return u'' - s = u'Character\n=====\nName: %s\n' % \ - self.get('name', u'') + if not self: + return '' + s = 'Character\n=====\nName: %s\n' % self.get('name', '') bio = self.get('biography') if bio: - s += u'Biography: %s\n' % bio[0] + s += 'Biography: %s\n' % bio[0] filmo = self.get('filmography') if filmo: - a_list = [x.get('long imdb canonical title', u'') - for x in filmo[:5]] - s += u'Last movies with this character: %s.\n' % u'; '.join(a_list) + a_list = [x.get('long imdb canonical title', '') for x in filmo[:5]] + s += 'Last movies with this character: %s.\n' % '; '.join(a_list) return s - - diff -pruN 5.1-1/imdb/cli.py 6.6-1/imdb/cli.py --- 5.1-1/imdb/cli.py 1970-01-01 00:00:00.000000000 +0000 +++ 6.6-1/imdb/cli.py 2018-08-05 13:36:02.000000000 +0000 @@ -0,0 +1,169 @@ +# Copyright 2017 H. Turgut Uyar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +""" +This module provides the command line interface for IMDbPY. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +from argparse import ArgumentParser + +from imdb import VERSION, IMDb + + +DEFAULT_RESULT_SIZE = 20 + + +def list_results(items, type_, n=None): + field = 'title' if type_ == 'movie' else 'name' + print(' # IMDb id %s' % field) + print('=== ======= %s' % ('=' * len(field),)) + for i, item in enumerate(items[:n]): + print('%(index)3d %(imdb_id)7s %(title)s' % { + 'index': i + 1, + 'imdb_id': getattr(item, type_ + 'ID'), + 'title': item['long imdb ' + field] + }) + + +def search_item(args): + connection = IMDb() + n = args.n if args.n is not None else DEFAULT_RESULT_SIZE + if args.type == 'keyword': + items = connection.search_keyword(args.key) + if args.first: + items = connection.get_keyword(items[0]) + list_results(items, type_='movie', n=n) + else: + print(' # keyword') + print('=== =======') + for i, keyword in enumerate(items[:n]): + print('%(index)3d %(kw)s' % {'index': i + 1, 'kw': keyword}) + else: + if args.type == 'movie': + items = connection.search_movie(args.key) + elif args.type == 'person': + items = connection.search_person(args.key) + elif args.type == 'character': + items = connection.search_character(args.key) + elif args.type == 'company': + items = connection.search_company(args.key) + + if args.first: + connection.update(items[0]) + print(items[0].summary()) + else: + list_results(items, type_=args.type, n=args.n) + + +def get_item(args): + connection = IMDb() + if args.type == 'keyword': + n = args.n if args.n is not None else DEFAULT_RESULT_SIZE + items = connection.get_keyword(args.key, results=n) + list_results(items, type_='movie') + else: + if args.type == 'movie': + item = connection.get_movie(args.key) + elif args.type == 'person': + item = connection.get_person(args.key) + elif args.type == 'character': + item = connection.get_character(args.key) + elif args.type == 'company': + item = connection.get_company(args.key) + print(item.summary()) + + +def list_ranking(items, n=None): + print(' # rating votes IMDb id title') + print('=== ====== ======= ======= =====') + n = n if n is not None else DEFAULT_RESULT_SIZE + for i, movie in enumerate(items[:n]): + print('%(index)3d %(rating)s %(votes)7s %(imdb_id)7s %(title)s' % { + 'index': i + 1, + 'rating': movie.get('rating'), + 'votes': movie.get('votes'), + 'imdb_id': movie.movieID, + 'title': movie.get('long imdb title') + }) + + +def get_top_movies(args): + connection = IMDb() + items = connection.get_top250_movies() + if args.first: + connection.update(items[0]) + print(items[0].summary()) + else: + list_ranking(items, n=args.n) + + +def get_bottom_movies(args): + connection = IMDb() + items = connection.get_bottom100_movies() + if args.first: + connection.update(items[0]) + print(items[0].summary()) + else: + list_ranking(items, n=args.n) + + +def make_parser(prog): + parser = ArgumentParser(prog) + parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION) + + command_parsers = parser.add_subparsers(metavar='command', dest='command') + command_parsers.required = True + + command_search_parser = command_parsers.add_parser('search', help='search for items') + command_search_parser.add_argument('type', help='type of item to search for', + choices=['movie', 'person', 'character', 'company', 'keyword']) + command_search_parser.add_argument('key', help='title or name of item to search for') + command_search_parser.add_argument('-n', type=int, help='number of items to list') + command_search_parser.add_argument('--first', action='store_true', help='display only the first result') + command_search_parser.set_defaults(func=search_item) + + command_get_parser = command_parsers.add_parser('get', help='retrieve information about an item') + command_get_parser.add_argument('type', help='type of item to retrieve', + choices=['movie', 'person', 'character', 'company', 'keyword']) + command_get_parser.add_argument('key', help='IMDb id (or keyword name) of item to retrieve') + command_get_parser.add_argument('-n', type=int, help='number of movies to list (only for keywords)') + command_get_parser.set_defaults(func=get_item) + + command_top_parser = command_parsers.add_parser('top', help='get top ranked movies') + command_top_parser.add_argument('-n', type=int, help='number of movies to list') + command_top_parser.add_argument('--first', action='store_true', help='display only the first result') + command_top_parser.set_defaults(func=get_top_movies) + + command_bottom_parser = command_parsers.add_parser('bottom', help='get bottom ranked movies') + command_bottom_parser.add_argument('-n', type=int, help='number of movies to list') + command_bottom_parser.add_argument('--first', action='store_true', help='display only the first result') + command_bottom_parser.set_defaults(func=get_bottom_movies) + + return parser + + +def main(argv=None): + argv = argv if argv is not None else sys.argv + parser = make_parser(prog='imdbpy') + arguments = parser.parse_args(argv[1:]) + arguments.func(arguments) + + +if __name__ == '__main__': + main() diff -pruN 5.1-1/imdb/Company.py 6.6-1/imdb/Company.py --- 5.1-1/imdb/Company.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/Company.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,37 +1,39 @@ -""" -company module (imdb package). +# Copyright 2008-2017 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides the company class, used to store information about a given company. - -Copyright 2008-2009 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +from __future__ import absolute_import, division, print_function, unicode_literals + from copy import deepcopy -from imdb.utils import analyze_company_name, build_company_name, \ - flatten, _Container, cmpCompanies +from imdb.utils import _Container +from imdb.utils import analyze_company_name, build_company_name, cmpCompanies, flatten class Company(_Container): """A company. - Every information about a company can be accessed as: + Every information about a company can be accessed as:: + companyObject['information'] + to get a list of the kind of information stored in a company object, use the keys() method; some useful aliases are defined (as "also known as" for the "akas" key); @@ -42,14 +44,15 @@ class Company(_Container): # Aliases for some not-so-intuitive keys. keys_alias = { - 'distributor': 'distributors', - 'special effects company': 'special effects companies', - 'other company': 'miscellaneous companies', - 'miscellaneous company': 'miscellaneous companies', - 'other companies': 'miscellaneous companies', - 'misc companies': 'miscellaneous companies', - 'misc company': 'miscellaneous companies', - 'production company': 'production companies'} + 'distributor': 'distributors', + 'special effects company': 'special effects companies', + 'other company': 'miscellaneous companies', + 'miscellaneous company': 'miscellaneous companies', + 'other companies': 'miscellaneous companies', + 'misc companies': 'miscellaneous companies', + 'misc company': 'miscellaneous companies', + 'production company': 'production companies' + } keys_tomodify_list = () @@ -71,23 +74,22 @@ class Company(_Container): *modFunct* -- function called returning text fields. """ name = kwds.get('name') - if name and not self.data.has_key('name'): + if name and 'name' not in self.data: self.set_name(name) self.companyID = kwds.get('companyID', None) - self.myName = kwds.get('myName', u'') + self.myName = kwds.get('myName', '') def _reset(self): """Reset the company object.""" self.companyID = None - self.myName = u'' + self.myName = '' def set_name(self, name): """Set the name of the company.""" - # XXX: convert name to unicode, if it's a plain string? # Company diverges a bit from other classes, being able # to directly handle its "notes". AND THAT'S PROBABLY A BAD IDEA! oname = name = name.strip() - notes = u'' + notes = '' if name.endswith(')'): fparidx = name.find('(') if fparidx != -1: @@ -102,14 +104,14 @@ class Company(_Container): def _additional_keys(self): """Valid keys to append to the data.keys() list.""" - if self.data.has_key('name'): + if 'name' in self.data: return ['long imdb name'] return [] def _getitem(self, key): """Handle special keys.""" - ## XXX: can a company have an imdbIndex? - if self.data.has_key('name'): + # XXX: can a company have an imdbIndex? + if 'name' in self.data: if key == 'long imdb name': return build_company_name(self.data) return None @@ -118,41 +120,42 @@ class Company(_Container): """Return the companyID.""" return self.companyID - def __nonzero__(self): + def __bool__(self): """The company is "false" if the self.data does not contain a name.""" # XXX: check the name and the companyID? - if self.data.get('name'): return 1 - return 0 + return bool(self.data.get('name')) def __contains__(self, item): """Return true if this company and the given Movie are related.""" - from Movie import Movie + from .Movie import Movie if isinstance(item, Movie): - for m in flatten(self.data, yieldDictKeys=1, scalar=Movie): + for m in flatten(self.data, yieldDictKeys=True, scalar=Movie): if item.isSame(m): - return 1 - return 0 + return True + elif isinstance(item, str): + return item in self.data + return False def isSameName(self, other): """Return true if two company have the same name and/or companyID.""" if not isinstance(other, self.__class__): - return 0 - if self.data.has_key('name') and \ - other.data.has_key('name') and \ + return False + if 'name' in self.data and \ + 'name' in other.data and \ build_company_name(self.data) == \ build_company_name(other.data): - return 1 + return True if self.accessSystem == other.accessSystem and \ self.companyID is not None and \ self.companyID == other.companyID: - return 1 - return 0 + return True + return False isSameCompany = isSameName def __deepcopy__(self, memo): """Return a deep copy of a company instance.""" - c = Company(name=u'', companyID=self.companyID, + c = Company(name='', companyID=self.companyID, myName=self.myName, myID=self.myID, data=deepcopy(self.data, memo), notes=self.notes, accessSystem=self.accessSystem, @@ -165,31 +168,25 @@ class Company(_Container): def __repr__(self): """String representation of a Company object.""" - r = '' % (self.companyID, - self.accessSystem, - self.get('long imdb name')) - if isinstance(r, unicode): r = r.encode('utf_8', 'replace') - return r + return '' % ( + self.companyID, self.accessSystem, self.get('long imdb name') + ) def __str__(self): """Simply print the short name.""" - return self.get('name', u'').encode('utf_8', 'replace') - - def __unicode__(self): - """Simply print the short title.""" - return self.get('name', u'') + return self.get('name', '') def summary(self): """Return a string with a pretty-printed summary for the company.""" - if not self: return u'' - s = u'Company\n=======\nName: %s\n' % \ - self.get('name', u'') + if not self: + return '' + s = 'Company\n=======\nName: %s\n' % self.get('name', '') for k in ('distributor', 'production company', 'miscellaneous company', - 'special effects company'): + 'special effects company'): d = self.get(k, [])[:5] - if not d: continue - s += u'Last movies from this company (%s): %s.\n' % \ - (k, u'; '.join([x.get('long imdb title', u'') for x in d])) + if not d: + continue + s += 'Last movies from this company (%s): %s.\n' % ( + k, '; '.join([x.get('long imdb title', '') for x in d]) + ) return s - - diff -pruN 5.1-1/imdb/_compat.py 6.6-1/imdb/_compat.py --- 5.1-1/imdb/_compat.py 2015-08-28 15:14:32.000000000 +0000 +++ 6.6-1/imdb/_compat.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,72 +0,0 @@ -""" -_compat module (imdb package). - -This module provides compatibility functions used by the imdb package -to deal with unusual environments. - -Copyright 2008-2010 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -""" - -# TODO: now we're heavily using the 'logging' module, which was not -# present in Python 2.2. To work in a Symbian environment, we -# need to create a fake 'logging' module (its functions may call -# the 'warnings' module, or do nothing at all). - - -import os -# If true, we're working on a Symbian device. -if os.name == 'e32': - # Replace os.path.expandvars and os.path.expanduser, if needed. - def _noact(x): - """Ad-hoc replacement for IMDbPY.""" - return x - try: - os.path.expandvars - except AttributeError: - os.path.expandvars = _noact - try: - os.path.expanduser - except AttributeError: - os.path.expanduser = _noact - - # time.strptime is missing, on Symbian devices. - import time - try: - time.strptime - except AttributeError: - import re - _re_web_time = re.compile(r'Episode dated (\d+) (\w+) (\d+)') - _re_ptdf_time = re.compile(r'\((\d+)-(\d+)-(\d+)\)') - _month2digit = {'January': '1', 'February': '2', 'March': '3', - 'April': '4', 'May': '5', 'June': '6', 'July': '7', - 'August': '8', 'September': '9', 'October': '10', - 'November': '11', 'December': '12'} - def strptime(s, format): - """Ad-hoc strptime replacement for IMDbPY.""" - try: - if format.startswith('Episode'): - res = _re_web_time.findall(s)[0] - return (int(res[2]), int(_month2digit[res[1]]), int(res[0]), - 0, 0, 0, 0, 1, 0) - else: - res = _re_ptdf_time.findall(s)[0] - return (int(res[0]), int(res[1]), int(res[2]), - 0, 0, 0, 0, 1, 0) - except: - raise ValueError('error in IMDbPY\'s ad-hoc strptime!') - time.strptime = strptime - diff -pruN 5.1-1/imdb/_exceptions.py 6.6-1/imdb/_exceptions.py --- 5.1-1/imdb/_exceptions.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/_exceptions.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,25 +1,25 @@ -""" -_exceptions module (imdb package). +# Copyright 2004-2017 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides the exception hierarchy used by the imdb package. - -Copyright 2004-2009 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +from __future__ import absolute_import, division, print_function, unicode_literals + import logging @@ -31,15 +31,15 @@ class IMDbError(Exception): """Initialize the exception and pass the message to the log system.""" # Every raised exception also dispatch a critical log. self._logger.critical('%s exception raised; args: %s; kwds: %s', - self.__class__.__name__, args, kwargs, - exc_info=True) + self.__class__.__name__, args, kwargs, exc_info=True) Exception.__init__(self, *args, **kwargs) + class IMDbDataAccessError(IMDbError): """Exception raised when is not possible to access needed data.""" pass + class IMDbParserError(IMDbError): """Exception raised when an error occurred parsing the data.""" pass - diff -pruN 5.1-1/imdb/helpers.py 6.6-1/imdb/helpers.py --- 5.1-1/imdb/helpers.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/helpers.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,52 +1,48 @@ -""" -helpers module (imdb package). +# Copyright 2006-2018 Davide Alberani +# 2012 Alberto Malagoli +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides functions not used directly by the imdb package, but useful for IMDbPY-based programs. - -Copyright 2006-2012 Davide Alberani - 2012 Alberto Malagoli - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ -# XXX: find better names for the functions in this modules. +# XXX: Find better names for the functions in this module. + +from __future__ import absolute_import, division, print_function, unicode_literals -import re import difflib -from cgi import escape import gettext +import re +from cgi import escape from gettext import gettext as _ -gettext.textdomain('imdbpy') # The modClearRefs can be used to strip names and titles references from # the strings in Movie and Person objects. -from imdb.utils import modClearRefs, re_titleRef, re_nameRef, \ - re_characterRef, _tagAttr, _Container, TAGS_TO_MODIFY -from imdb import IMDb, imdbURL_movie_base, imdbURL_person_base, \ - imdbURL_character_base - -import imdb.locale +from imdb import IMDb, imdbURL_character_base, imdbURL_movie_base, imdbURL_person_base +from imdb.Character import Character +from imdb.Company import Company from imdb.linguistics import COUNTRY_LANG from imdb.Movie import Movie from imdb.Person import Person -from imdb.Character import Character -from imdb.Company import Company -from imdb.parser.http.utils import re_entcharrefssub, entcharrefs, \ - subXMLRefs, subSGMLRefs -from imdb.parser.http.bsouplxml.etree import BeautifulSoup +from imdb.utils import _tagAttr, re_characterRef, re_nameRef, re_titleRef +from imdb.utils import TAGS_TO_MODIFY + + +gettext.textdomain('imdbpy') # An URL, more or less. @@ -60,13 +56,14 @@ def makeCgiPrintEncoding(encoding): """Encode the given string using the %s encoding, and replace chars outside the given charset with XML char references.""" % encoding s = escape(s, quote=1) - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode(encoding, 'xmlcharrefreplace') return s return cgiPrint -# cgiPrint uses the latin_1 encoding. -cgiPrint = makeCgiPrintEncoding('latin_1') + +# cgiPrint uses the utf8 encoding. +cgiPrint = makeCgiPrintEncoding('utf8') # Regular expression for %(varname)s substitutions. re_subst = re.compile(r'%\((.+?)\)s') @@ -85,7 +82,7 @@ def makeTextNotes(replaceTxtNotes): of the makeObject2Txt function.""" def _replacer(s): outS = replaceTxtNotes - if not isinstance(s, (unicode, str)): + if not isinstance(s, str): return s ssplit = s.split('::', 1) text = ssplit[0] @@ -98,12 +95,14 @@ def makeTextNotes(replaceTxtNotes): keysDict['notes'] = True outS = outS.replace('%(notes)s', ssplit[1]) else: - outS = outS.replace('%(notes)s', u'') + outS = outS.replace('%(notes)s', '') + def _excludeFalseConditionals(matchobj): # Return an empty string if the conditional is false/empty. if matchobj.group(1) in keysDict: return matchobj.group(2) - return u'' + return '' + while re_conditional.search(outS): outS = re_conditional.sub(_excludeFalseConditionals, outS) return outS @@ -111,8 +110,8 @@ def makeTextNotes(replaceTxtNotes): def makeObject2Txt(movieTxt=None, personTxt=None, characterTxt=None, - companyTxt=None, joiner=' / ', - applyToValues=lambda x: x, _recurse=True): + companyTxt=None, joiner=' / ', + applyToValues=lambda x: x, _recurse=True): """"Return a function useful to pretty-print Movie, Person, Character and Company instances. @@ -133,23 +132,25 @@ def makeObject2Txt(movieTxt=None, person characterTxt = '%(long imdb name)s' if companyTxt is None: companyTxt = '%(long imdb name)s' + def object2txt(obj, _limitRecursion=None): """Pretty-print objects.""" # Prevent unlimited recursion. if _limitRecursion is None: _limitRecursion = 0 elif _limitRecursion > 5: - return u'' + return '' _limitRecursion += 1 if isinstance(obj, (list, tuple)): return joiner.join([object2txt(o, _limitRecursion=_limitRecursion) for o in obj]) elif isinstance(obj, dict): # XXX: not exactly nice, neither useful, I fear. - return joiner.join([u'%s::%s' % - (object2txt(k, _limitRecursion=_limitRecursion), - object2txt(v, _limitRecursion=_limitRecursion)) - for k, v in obj.items()]) + return joiner.join( + ['%s::%s' % (object2txt(k, _limitRecursion=_limitRecursion), + object2txt(v, _limitRecursion=_limitRecursion)) + for k, v in list(obj.items())] + ) objData = {} if isinstance(obj, Movie): objData['movieID'] = obj.movieID @@ -165,6 +166,7 @@ def makeObject2Txt(movieTxt=None, person outs = companyTxt else: return obj + def _excludeFalseConditionals(matchobj): # Return an empty string if the conditional is false/empty. condition = matchobj.group(1) @@ -172,61 +174,64 @@ def makeObject2Txt(movieTxt=None, person if proceed: return matchobj.group(2) else: - return u'' - return matchobj.group(2) + return '' while re_conditional.search(outs): outs = re_conditional.sub(_excludeFalseConditionals, outs) for key in re_subst.findall(outs): value = obj.get(key) or getattr(obj, key, None) - if not isinstance(value, (unicode, str)): + if not isinstance(value, str): if not _recurse: if value: - value = unicode(value) + value = str(value) if value: value = object2txt(value, _limitRecursion=_limitRecursion) elif value: - value = applyToValues(unicode(value)) + value = applyToValues(str(value)) if not value: - value = u'' - elif not isinstance(value, (unicode, str)): - value = unicode(value) - outs = outs.replace(u'%(' + key + u')s', value) + value = '' + elif not isinstance(value, str): + value = str(value) + outs = outs.replace('%(' + key + ')s', value) return outs return object2txt -def makeModCGILinks(movieTxt, personTxt, characterTxt=None, - encoding='latin_1'): +def makeModCGILinks(movieTxt, personTxt, characterTxt=None, encoding='utf8'): """Make a function used to pretty-print movies and persons refereces; movieTxt and personTxt are the strings used for the substitutions. movieTxt must contains %(movieID)s and %(title)s, while personTxt must contains %(personID)s and %(name)s and characterTxt %(characterID)s and %(name)s; characterTxt is optional, for backward compatibility.""" _cgiPrint = makeCgiPrintEncoding(encoding) + def modCGILinks(s, titlesRefs, namesRefs, characterRefs=None): """Substitute movies and persons references.""" - if characterRefs is None: characterRefs = {} + if characterRefs is None: + characterRefs = {} + # XXX: look ma'... more nested scopes! def _replaceMovie(match): to_replace = match.group(1) item = titlesRefs.get(to_replace) if item: movieID = item.movieID - to_replace = movieTxt % {'movieID': movieID, - 'title': unicode(_cgiPrint(to_replace), - encoding, - 'xmlcharrefreplace')} + to_replace = movieTxt % { + 'movieID': movieID, + 'title': str(_cgiPrint(to_replace), encoding, 'xmlcharrefreplace') + } return to_replace + def _replacePerson(match): to_replace = match.group(1) item = namesRefs.get(to_replace) if item: personID = item.personID - to_replace = personTxt % {'personID': personID, - 'name': unicode(_cgiPrint(to_replace), - encoding, - 'xmlcharrefreplace')} + to_replace = personTxt % { + 'personID': personID, + 'name': str(_cgiPrint(to_replace), encoding, 'xmlcharrefreplace') + } return to_replace + def _replaceCharacter(match): to_replace = match.group(1) if characterTxt is None: @@ -236,10 +241,10 @@ def makeModCGILinks(movieTxt, personTxt, characterID = item.characterID if characterID is None: return to_replace - to_replace = characterTxt % {'characterID': characterID, - 'name': unicode(_cgiPrint(to_replace), - encoding, - 'xmlcharrefreplace')} + to_replace = characterTxt % { + 'characterID': characterID, + 'name': str(_cgiPrint(to_replace), encoding, 'xmlcharrefreplace') + } return to_replace s = s.replace('<', '<').replace('>', '>') s = _re_hrefsub(r'\1', s) @@ -252,47 +257,22 @@ def makeModCGILinks(movieTxt, personTxt, modCGILinks.characterTxt = characterTxt return modCGILinks + # links to the imdb.com web site. _movieTxt = '%(title)s' _personTxt = '%(name)s' _characterTxt = '%(name)s' modHtmlLinks = makeModCGILinks(movieTxt=_movieTxt, personTxt=_personTxt, - characterTxt=_characterTxt) + characterTxt=_characterTxt) modHtmlLinksASCII = makeModCGILinks(movieTxt=_movieTxt, personTxt=_personTxt, characterTxt=_characterTxt, encoding='ascii') -everyentcharrefs = entcharrefs.copy() -for k, v in {'lt':u'<','gt':u'>','amp':u'&','quot':u'"','apos':u'\''}.items(): - everyentcharrefs[k] = v - everyentcharrefs['#%s' % ord(v)] = v -everyentcharrefsget = everyentcharrefs.get -re_everyentcharrefs = re.compile('&(%s|\#160|\#\d{1,5});' % - '|'.join(map(re.escape, everyentcharrefs))) -re_everyentcharrefssub = re_everyentcharrefs.sub - -def _replAllXMLRef(match): - """Replace the matched XML reference.""" - ref = match.group(1) - value = everyentcharrefsget(ref) - if value is None: - if ref[0] == '#': - return unichr(int(ref[1:])) - else: - return ref - return value - -def subXMLHTMLSGMLRefs(s): - """Return the given string with XML/HTML/SGML entity and char references - replaced.""" - return re_everyentcharrefssub(_replAllXMLRef, s) - - def sortedSeasons(m): """Return a sorted list of seasons of the given series.""" - seasons = m.get('episodes', {}).keys() + seasons = list(m.get('episodes', {}).keys()) seasons.sort() return seasons @@ -308,7 +288,7 @@ def sortedEpisodes(m, season=None): if not isinstance(season, (tuple, list)): seasons = [season] for s in seasons: - eps_indx = m.get('episodes', {}).get(s, {}).keys() + eps_indx = list(m.get('episodes', {}).get(s, {}).keys()) eps_indx.sort() for e in eps_indx: episodes.append(m['episodes'][s][e]) @@ -317,14 +297,18 @@ def sortedEpisodes(m, season=None): # Idea and portions of the code courtesy of none none (dclist at gmail.com) _re_imdbIDurl = re.compile(r'\b(nm|tt|ch|co)([0-9]{7})\b') + + def get_byURL(url, info=None, args=None, kwds=None): """Return a Movie, Person, Character or Company object for the given URL; info is the info set to retrieve, args and kwds are respectively a list and a dictionary or arguments to initialize the data access system. Returns None if unable to correctly parse the url; can raise exceptions if unable to retrieve the data.""" - if args is None: args = [] - if kwds is None: kwds = {} + if args is None: + args = [] + if kwds is None: + kwds = {} ia = IMDb(*args, **kwds) match = _re_imdbIDurl.search(url) if not match: @@ -351,15 +335,7 @@ def fullSizeCoverURL(obj): or None otherwise. This function is obsolete: the same information are available as keys: 'full-size cover url' and 'full-size headshot', respectively for movies and persons/characters.""" - if isinstance(obj, Movie): - coverUrl = obj.get('cover url') - elif isinstance(obj, (Person, Character)): - coverUrl = obj.get('headshot') - else: - coverUrl = obj - if not coverUrl: - return None - return _Container._re_fullsizeURL.sub('', coverUrl) + return obj.get_fullsizeURL() def keyToXML(key): @@ -382,9 +358,10 @@ _MAP_TOP_OBJ = { } # Tags to be converted to lists. -_TAGS_TO_LIST = dict([(x[0], None) for x in TAGS_TO_MODIFY.values()]) +_TAGS_TO_LIST = dict([(x[0], None) for x in list(TAGS_TO_MODIFY.values())]) _TAGS_TO_LIST.update(_MAP_TOP_OBJ) + def tagToKey(tag): """Return the name of the tag, taking it from the 'key' attribute, if present.""" @@ -393,7 +370,7 @@ def tagToKey(tag): if tag.get('keytype') == 'int': keyAttr = int(keyAttr) return keyAttr - return tag.name + return tag.tag def _valueWithType(tag, tagValue): @@ -408,11 +385,11 @@ def _valueWithType(tag, tagValue): # Extra tags to get (if values were not already read from title/name). _titleTags = ('imdbindex', 'kind', 'year') -_nameTags = ('imdbindex') +_nameTags = ('imdbindex',) _companyTags = ('imdbindex', 'country') -def parseTags(tag, _topLevel=True, _as=None, _infoset2keys=None, - _key2infoset=None): + +def parseTags(tag, _topLevel=True, _as=None, _infoset2keys=None, _key2infoset=None): """Recursively parse a tree of tags.""" # The returned object (usually a _Container subclass, but it can # be a string, an int, a float, a list or a dictionary). @@ -422,20 +399,19 @@ def parseTags(tag, _topLevel=True, _as=N if _key2infoset is None: _key2infoset = {} name = tagToKey(tag) - firstChild = tag.find(recursive=False) - tagStr = (tag.string or u'').strip() + firstChild = (tag.getchildren() or [None])[0] + tagStr = (tag.text or '').strip() if not tagStr and name == 'item': # Handles 'item' tags containing text and a 'notes' sub-tag. - tagContent = tag.contents[0] - if isinstance(tagContent, BeautifulSoup.NavigableString): - tagStr = (unicode(tagContent) or u'').strip() - tagType = tag.get('type') + tagContent = tag.getchildren() + if tagContent and tagContent[0].text: + tagStr = (tagContent[0].text or '').strip() infoset = tag.get('infoset') if infoset: _key2infoset[name] = infoset _infoset2keys.setdefault(infoset, []).append(name) # Here we use tag.name to avoid tags like - if tag.name in _MAP_TOP_OBJ: + if tag.tag in _MAP_TOP_OBJ: # One of the subclasses of _Container. item = _MAP_TOP_OBJ[name]() itemAs = tag.get('access-system') @@ -450,67 +426,67 @@ def parseTags(tag, _topLevel=True, _as=N if name == 'movie': item.movieID = theID tagsToGet = _titleTags - theTitle = tag.find('title', recursive=False) - if tag.title: - item.set_title(tag.title.string) - tag.title.extract() + ttitle = tag.find('title') + if ttitle is not None: + item.set_title(ttitle.text) + tag.remove(ttitle) else: if name == 'person': item.personID = theID tagsToGet = _nameTags - theName = tag.find('long imdb canonical name', recursive=False) + theName = tag.find('long imdb canonical name') if not theName: - theName = tag.find('name', recursive=False) + theName = tag.find('name') elif name == 'character': item.characterID = theID tagsToGet = _nameTags - theName = tag.find('name', recursive=False) + theName = tag.find('name') elif name == 'company': item.companyID = theID tagsToGet = _companyTags - theName = tag.find('name', recursive=False) - if theName: - item.set_name(theName.string) - if theName: - theName.extract() + theName = tag.find('name') + if theName is not None: + item.set_name(theName.text) + tag.remove(theName) for t in tagsToGet: if t in item.data: continue - dataTag = tag.find(t, recursive=False) - if dataTag: - item.data[tagToKey(dataTag)] = _valueWithType(dataTag, - dataTag.string) - if tag.notes: - item.notes = tag.notes.string - tag.notes.extract() - episodeOf = tag.find('episode-of', recursive=False) - if episodeOf: + dataTag = tag.find(t) + if dataTag is not None: + item.data[tagToKey(dataTag)] = _valueWithType(dataTag, dataTag.text) + notesTag = tag.find('notes') + if notesTag is not None: + item.notes = notesTag.text + tag.remove(notesTag) + episodeOf = tag.find('episode-of') + if episodeOf is not None: item.data['episode of'] = parseTags(episodeOf, _topLevel=False, - _as=_as, _infoset2keys=_infoset2keys, - _key2infoset=_key2infoset) - episodeOf.extract() - cRole = tag.find('current-role', recursive=False) - if cRole: + _as=_as, _infoset2keys=_infoset2keys, + _key2infoset=_key2infoset) + tag.remove(episodeOf) + cRole = tag.find('current-role') + if cRole is not None: cr = parseTags(cRole, _topLevel=False, _as=_as, - _infoset2keys=_infoset2keys, _key2infoset=_key2infoset) + _infoset2keys=_infoset2keys, _key2infoset=_key2infoset) item.currentRole = cr - cRole.extract() + tag.remove(cRole) # XXX: big assumption, here. What about Movie instances used # as keys in dictionaries? What about other keys (season and # episode number, for example?) if not _topLevel: - #tag.extract() + # tag.extract() return item _adder = lambda key, value: item.data.update({key: value}) elif tagStr: - if tag.notes: - notes = (tag.notes.string or u'').strip() + tagNotes = tag.find('notes') + if tagNotes is not None: + notes = (tagNotes.text or '').strip() if notes: - tagStr += u'::%s' % notes + tagStr += '::%s' % notes else: tagStr = _valueWithType(tag, tagStr) return tagStr - elif firstChild: + elif firstChild is not None: firstChildName = tagToKey(firstChild) if firstChildName in _TAGS_TO_LIST: item = [] @@ -521,49 +497,44 @@ def parseTags(tag, _topLevel=True, _as=N else: item = {} _adder = lambda key, value: item.update({name: value}) - for subTag in tag(recursive=False): + for subTag in tag.getchildren(): subTagKey = tagToKey(subTag) # Exclude dinamically generated keys. - if tag.name in _MAP_TOP_OBJ and subTagKey in item._additional_keys(): + if tag.tag in _MAP_TOP_OBJ and subTagKey in item._additional_keys(): continue subItem = parseTags(subTag, _topLevel=False, _as=_as, - _infoset2keys=_infoset2keys, _key2infoset=_key2infoset) + _infoset2keys=_infoset2keys, _key2infoset=_key2infoset) if subItem: _adder(subTagKey, subItem) if _topLevel and name in _MAP_TOP_OBJ: # Add information about 'info sets', but only to the top-level object. item.infoset2keys = _infoset2keys item.key2infoset = _key2infoset - item.current_info = _infoset2keys.keys() + item.current_info = list(_infoset2keys.keys()) return item def parseXML(xml): """Parse a XML string, returning an appropriate object (usually an instance of a subclass of _Container.""" - xmlObj = BeautifulSoup.BeautifulStoneSoup(xml, - convertEntities=BeautifulSoup.BeautifulStoneSoup.XHTML_ENTITIES) - if xmlObj: - mainTag = xmlObj.find() - if mainTag: - return parseTags(mainTag) - return None + import lxml.etree + return parseTags(lxml.etree.fromstring(xml)) _re_akas_lang = re.compile('(?:[(])([a-zA-Z]+?)(?: title[)])') _re_akas_country = re.compile('\(.*?\)') + # akasLanguages, sortAKAsBySimilarity and getAKAsInLanguage code # copyright of Alberto Malagoli (refactoring by Davide Alberani). def akasLanguages(movie): """Given a movie, return a list of tuples in (lang, AKA) format; lang can be None, if unable to detect.""" lang_and_aka = [] - akas = set((movie.get('akas') or []) + - (movie.get('akas from release info') or [])) + akas = set((movie.get('akas') or []) + (movie.get('akas from release info') or [])) for aka in akas: # split aka - aka = aka.encode('utf8').split('::') + aka = aka.split('::') # sometimes there is no countries information if len(aka) == 2: # search for something like "(... title)" where ... is a language @@ -579,7 +550,7 @@ def akasLanguages(movie): language = COUNTRY_LANG.get(country) else: language = None - lang_and_aka.append((language, aka[0].decode('utf8'))) + lang_and_aka.append((language, aka[0])) return lang_and_aka @@ -594,17 +565,13 @@ def sortAKAsBySimilarity(movie, title, _ # estimate string distance between current title and given title m_title = movie['title'].lower() l_title = title.lower() - if isinstance(l_title, unicode): - l_title = l_title.encode('utf8') scores = [] - score = difflib.SequenceMatcher(None, m_title.encode('utf8'), l_title).ratio() + score = difflib.SequenceMatcher(None, m_title, l_title).ratio() # set original title and corresponding score as the best match for given title scores.append((score, movie['title'], None)) for language, aka in akasLanguages(movie): # estimate string distance between current title and given title m_title = aka.lower() - if isinstance(m_title, unicode): - m_title = m_title.encode('utf8') score = difflib.SequenceMatcher(None, m_title, l_title).ratio() # if current language is the same as the given one, increase score if _preferredLang and _preferredLang == language: @@ -626,15 +593,9 @@ def getAKAsInLanguage(movie, lang, _sear akas.append(aka) if _searchedTitle: scores = [] - if isinstance(_searchedTitle, unicode): - _searchedTitle = _searchedTitle.encode('utf8') for aka in akas: - m_aka = aka - if isinstance(m_aka): - m_aka = m_aka.encode('utf8') - scores.append(difflib.SequenceMatcher(None, m_aka.lower(), - _searchedTitle.lower()), aka) + scores.append(difflib.SequenceMatcher(None, aka.lower(), + _searchedTitle.lower()), aka) scores.sort(reverse=True) akas = [x[1] for x in scores] return akas - diff -pruN 5.1-1/imdb/__init__.py 6.6-1/imdb/__init__.py --- 5.1-1/imdb/__init__.py 2016-11-13 11:36:15.000000000 +0000 +++ 6.6-1/imdb/__init__.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,88 +1,104 @@ -""" -imdb package. +# Copyright 2004-2018 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -This package can be used to retrieve information about a movie or -a person from the IMDb database. -It can fetch data through different media (e.g.: the IMDb web pages, -a SQL database, etc.) - -Copyright 2004-2016 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +This package can be used to retrieve information about a movie or a person +from the IMDb database. It can fetch data through different media such as +the IMDb web pages, or a SQL database. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + __all__ = ['IMDb', 'IMDbError', 'Movie', 'Person', 'Character', 'Company', - 'available_access_systems'] -__version__ = VERSION = '5.1' + 'available_access_systems'] +__version__ = VERSION = '6.6' -# Import compatibility module (importing it is enough). -import _compat -import sys, os, ConfigParser, logging -from types import MethodType +import logging +import os +import sys +from pkgutil import find_loader +from types import MethodType, FunctionType -from imdb import Movie, Person, Character, Company import imdb._logging -from imdb._exceptions import IMDbError, IMDbDataAccessError, IMDbParserError -from imdb.utils import build_title, build_name, build_company_name +from imdb._exceptions import IMDbDataAccessError, IMDbError +from imdb import Character, Company, Movie, Person +from imdb.utils import build_company_name, build_name, build_title + + +PY2 = sys.hexversion < 0x3000000 + + +if PY2: + import ConfigParser as configparser +else: + import configparser + +_imdb_logger = logging.getLogger('imdbpy') _aux_logger = logging.getLogger('imdbpy.aux') # URLs of the main pages for movies, persons, characters and queries. -imdbURL_base = 'http://akas.imdb.com/' +imdbURL_base = 'http://www.imdb.com/' # NOTE: the urls below will be removed in a future version. # please use the values in the 'urls' attribute # of the IMDbBase subclass instance. -# http://akas.imdb.com/title/ +# http://www.imdb.com/title/ imdbURL_movie_base = '%stitle/' % imdbURL_base -# http://akas.imdb.com/title/tt%s/ +# http://www.imdb.com/title/tt%s/ imdbURL_movie_main = imdbURL_movie_base + 'tt%s/' -# http://akas.imdb.com/name/ +# http://www.imdb.com/name/ imdbURL_person_base = '%sname/' % imdbURL_base -# http://akas.imdb.com/name/nm%s/ +# http://www.imdb.com/name/nm%s/ imdbURL_person_main = imdbURL_person_base + 'nm%s/' -# http://akas.imdb.com/character/ +# http://www.imdb.com/character/ imdbURL_character_base = '%scharacter/' % imdbURL_base -# http://akas.imdb.com/character/ch%s/ +# http://www.imdb.com/character/ch%s/ imdbURL_character_main = imdbURL_character_base + 'ch%s/' -# http://akas.imdb.com/company/ +# http://www.imdb.com/company/ imdbURL_company_base = '%scompany/' % imdbURL_base -# http://akas.imdb.com/company/co%s/ +# http://www.imdb.com/company/co%s/ imdbURL_company_main = imdbURL_company_base + 'co%s/' -# http://akas.imdb.com/keyword/%s/ +# http://www.imdb.com/keyword/%s/ imdbURL_keyword_main = imdbURL_base + 'keyword/%s/' -# http://akas.imdb.com/chart/top +# http://www.imdb.com/chart/top imdbURL_top250 = imdbURL_base + 'chart/top' -# http://akas.imdb.com/chart/bottom +# http://www.imdb.com/chart/bottom imdbURL_bottom100 = imdbURL_base + 'chart/bottom' -# http://akas.imdb.com/find?%s +# http://www.imdb.com/find?%s imdbURL_find = imdbURL_base + 'find?%s' # Name of the configuration file. confFileName = 'imdbpy.cfg' -class ConfigParserWithCase(ConfigParser.ConfigParser): + +class ConfigParserWithCase(configparser.ConfigParser): """A case-sensitive parser for configuration files.""" def __init__(self, defaults=None, confFile=None, *args, **kwds): """Initialize the parser. *defaults* -- defaults values. *confFile* -- the file (or list of files) to parse.""" - ConfigParser.ConfigParser.__init__(self, defaults=defaults) + if PY2: + configparser.ConfigParser.__init__(self, defaults=defaults) + else: + super(configparser.ConfigParser, self).__init__(defaults=defaults) if confFile is None: dotFileName = '.' + confFileName # Current and home directory. @@ -94,16 +110,15 @@ class ConfigParserWithCase(ConfigParser. sep = getattr(os.path, 'sep', '/') # /etc/ and /etc/conf.d/ confFile.append(os.path.join(sep, 'etc', confFileName)) - confFile.append(os.path.join(sep, 'etc', 'conf.d', - confFileName)) + confFile.append(os.path.join(sep, 'etc', 'conf.d', confFileName)) else: # etc subdirectory of sys.prefix, for non-unix systems. confFile.append(os.path.join(sys.prefix, 'etc', confFileName)) for fname in confFile: try: self.read(fname) - except (ConfigParser.MissingSectionHeaderError, - ConfigParser.ParsingError), e: + except (configparser.MissingSectionHeaderError, + configparser.ParsingError) as e: _aux_logger.warn('Troubles reading config file: %s' % e) # Stop at the first valid file. if self.has_section('imdbpy'): @@ -115,19 +130,18 @@ class ConfigParserWithCase(ConfigParser. def _manageValue(self, value): """Custom substitutions for values.""" - if not isinstance(value, (str, unicode)): + if not isinstance(value, str): return value vlower = value.lower() - if vlower in self._boolean_states: - return self._boolean_states[vlower] + if vlower in ('1', 'on', 'false', '0', 'off', 'yes', 'no', 'true'): + return self._convert_to_boolean(vlower) elif vlower == 'none': return None return value def get(self, section, option, *args, **kwds): """Return the value of an option from a given section.""" - value = ConfigParser.ConfigParser.get(self, section, option, - *args, **kwds) + value = configparser.ConfigParser.get(self, section, option, *args, **kwds) return self._manageValue(value) def items(self, section, *args, **kwds): @@ -135,7 +149,7 @@ class ConfigParserWithCase(ConfigParser. given section.""" if section != 'DEFAULT' and not self.has_section(section): return [] - keys = ConfigParser.ConfigParser.options(self, section) + keys = configparser.ConfigParser.options(self, section) return [(k, self.get(section, k, *args, **kwds)) for k in keys] def getDict(self, section): @@ -159,10 +173,8 @@ def IMDb(accessSystem=None, *arguments, accessSystem = 'http' kwds.update(keywords) keywords = kwds - except Exception, e: - import logging - logging.getLogger('imdbpy').warn('Unable to read configuration' \ - ' file; complete error: %s' % e) + except Exception as e: + _imdb_logger.warn('Unable to read configuration file; complete error: %s' % e) # It just LOOKS LIKE a bad habit: we tried to read config # options from some files, but something is gone horribly # wrong: ignore everything and pretend we were called with @@ -177,51 +189,31 @@ def IMDb(accessSystem=None, *arguments, try: import logging.config logging.config.fileConfig(os.path.expanduser(logCfg)) - except Exception, e: - logging.getLogger('imdbpy').warn('unable to read logger ' \ - 'config: %s' % e) - if accessSystem in ('httpThin', 'webThin', 'htmlThin'): - logging.warn('httpThin was removed since IMDbPY 4.8') - accessSystem = 'http' - if accessSystem in ('http', 'web', 'html'): - from parser.http import IMDbHTTPAccessSystem + except Exception as e: + _imdb_logger.warn('unable to read logger config: %s' % e) + if accessSystem in ('http', 'https', 'web', 'html'): + from .parser.http import IMDbHTTPAccessSystem return IMDbHTTPAccessSystem(*arguments, **keywords) - elif accessSystem in ('mobile',): - from parser.mobile import IMDbMobileAccessSystem - return IMDbMobileAccessSystem(*arguments, **keywords) - elif accessSystem in ('local', 'files'): - # The local access system was removed since IMDbPY 4.2. - raise IMDbError('the local access system was removed since IMDbPY 4.2') + if accessSystem in ('s3', 's3dataset', 'imdbws'): + from .parser.s3 import IMDbS3AccessSystem + return IMDbS3AccessSystem(*arguments, **keywords) elif accessSystem in ('sql', 'db', 'database'): try: - from parser.sql import IMDbSqlAccessSystem + from .parser.sql import IMDbSqlAccessSystem except ImportError: raise IMDbError('the sql access system is not installed') return IMDbSqlAccessSystem(*arguments, **keywords) else: - raise IMDbError('unknown kind of data access system: "%s"' \ - % accessSystem) + raise IMDbError('unknown kind of data access system: "%s"' % accessSystem) def available_access_systems(): """Return the list of available data access systems.""" asList = [] - # XXX: trying to import modules is a good thing? - try: - from parser.http import IMDbHTTPAccessSystem + if find_loader('imdb.parser.http') is not None: asList.append('http') - except ImportError: - pass - try: - from parser.mobile import IMDbMobileAccessSystem - asList.append('mobile') - except ImportError: - pass - try: - from parser.sql import IMDbSqlAccessSystem + if find_loader('imdb.parser.sql') is not None: asList.append('sql') - except ImportError: - pass return asList @@ -233,6 +225,7 @@ def available_access_systems(): # and search_character() methods is always safer. encoding = getattr(sys.stdin, 'encoding', '') or sys.getdefaultencoding() + class IMDbBase: """The base class used to search for a movie/person/character and to get a Movie/Person/Character object. @@ -244,14 +237,11 @@ class IMDbBase: # in the subclasses). accessSystem = 'UNKNOWN' - # Top-level logger for IMDbPY. - _imdb_logger = logging.getLogger('imdbpy') - # Whether to re-raise caught exceptions or not. _reraise_exceptions = False def __init__(self, defaultModFunct=None, results=20, keywordsResults=100, - *arguments, **keywords): + *arguments, **keywords): """Initialize the access system. If specified, defaultModFunct is the function used by default by the Person, Movie and Character objects, when @@ -285,30 +275,30 @@ class IMDbBase: imdbURL_base = 'http://%s' % imdbURL_base if not imdbURL_base.endswith('/'): imdbURL_base = '%s/' % imdbURL_base - # http://akas.imdb.com/title/ - imdbURL_movie_base='%stitle/' % imdbURL_base - # http://akas.imdb.com/title/tt%s/ - imdbURL_movie_main=imdbURL_movie_base + 'tt%s/' - # http://akas.imdb.com/name/ - imdbURL_person_base='%sname/' % imdbURL_base - # http://akas.imdb.com/name/nm%s/ - imdbURL_person_main=imdbURL_person_base + 'nm%s/' - # http://akas.imdb.com/character/ - imdbURL_character_base='%scharacter/' % imdbURL_base - # http://akas.imdb.com/character/ch%s/ - imdbURL_character_main=imdbURL_character_base + 'ch%s/' - # http://akas.imdb.com/company/ - imdbURL_company_base='%scompany/' % imdbURL_base - # http://akas.imdb.com/company/co%s/ - imdbURL_company_main=imdbURL_company_base + 'co%s/' - # http://akas.imdb.com/keyword/%s/ - imdbURL_keyword_main=imdbURL_base + 'keyword/%s/' - # http://akas.imdb.com/chart/top - imdbURL_top250=imdbURL_base + 'chart/top' - # http://akas.imdb.com/chart/bottom - imdbURL_bottom100=imdbURL_base + 'chart/bottom' - # http://akas.imdb.com/find?%s - imdbURL_find=imdbURL_base + 'find?%s' + # http://www.imdb.com/title/ + imdbURL_movie_base = '%stitle/' % imdbURL_base + # http://www.imdb.com/title/tt%s/ + imdbURL_movie_main = imdbURL_movie_base + 'tt%s/' + # http://www.imdb.com/name/ + imdbURL_person_base = '%sname/' % imdbURL_base + # http://www.imdb.com/name/nm%s/ + imdbURL_person_main = imdbURL_person_base + 'nm%s/' + # http://www.imdb.com/character/ + imdbURL_character_base = '%scharacter/' % imdbURL_base + # http://www.imdb.com/character/ch%s/ + imdbURL_character_main = imdbURL_character_base + 'ch%s/' + # http://www.imdb.com/company/ + imdbURL_company_base = '%scompany/' % imdbURL_base + # http://www.imdb.com/company/co%s/ + imdbURL_company_main = imdbURL_company_base + 'co%s/' + # http://www.imdb.com/keyword/%s/ + imdbURL_keyword_main = imdbURL_base + 'keyword/%s/' + # http://www.imdb.com/chart/top + imdbURL_top250 = imdbURL_base + 'chart/top' + # http://www.imdb.com/chart/bottom + imdbURL_bottom100 = imdbURL_base + 'chart/bottom' + # http://www.imdb.com/find?%s + imdbURL_find = imdbURL_base + 'find?%s' self.urls = dict( movie_base=imdbURL_movie_base, movie_main=imdbURL_movie_main, @@ -371,7 +361,7 @@ class IMDbBase: for name in dir(self.__class__): if name.startswith(prefname) and name not in excludes: member = getattr(self.__class__, name) - if isinstance(member, MethodType): + if isinstance(member, (MethodType, FunctionType)): infoset.append(name[preflen:].replace('_', ' ')) return infoset @@ -428,10 +418,6 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 20 - # XXX: I suppose it will be much safer if the user provides - # an unicode string... this is just a guess. - if not isinstance(title, unicode): - title = unicode(title, encoding, 'replace') if not _episodes: res = self._search_movie(title, results) else: @@ -452,8 +438,7 @@ class IMDbBase: this method searches only for titles of tv (mini) series' episodes.""" return self.search_movie(title, results=results, _episodes=True) - def get_person(self, personID, info=Person.Person.default_info, - modFunct=None): + def get_person(self, personID, info=Person.Person.default_info, modFunct=None): """Return a Person object for the given personID. The personID is something used to univocally identify a person; @@ -466,8 +451,7 @@ class IMDbBase: object when accessing its text fields (like 'mini biography').""" personID = self._normalize_personID(personID) personID = self._get_real_personID(personID) - person = Person.Person(personID=personID, - accessSystem=self.accessSystem) + person = Person.Person(personID=personID, accessSystem=self.accessSystem) modFunct = modFunct or self._defModFunct if modFunct is not None: person.set_mod_funct(modFunct) @@ -490,15 +474,13 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 20 - if not isinstance(name, unicode): - name = unicode(name, encoding, 'replace') res = self._search_person(name, results) return [Person.Person(personID=self._get_real_personID(pi), data=pd, modFunct=self._defModFunct, accessSystem=self.accessSystem) for pi, pd in res][:results] def get_character(self, characterID, info=Character.Character.default_info, - modFunct=None): + modFunct=None): """Return a Character object for the given characterID. The characterID is something used to univocally identify a character; @@ -512,7 +494,7 @@ class IMDbBase: characterID = self._normalize_characterID(characterID) characterID = self._get_real_characterID(characterID) character = Character.Character(characterID=characterID, - accessSystem=self.accessSystem) + accessSystem=self.accessSystem) modFunct = modFunct or self._defModFunct if modFunct is not None: character.set_mod_funct(modFunct) @@ -535,8 +517,6 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 20 - if not isinstance(name, unicode): - name = unicode(name, encoding, 'replace') res = self._search_character(name, results) return [Character.Character(characterID=self._get_real_characterID(pi), data=pd, modFunct=self._defModFunct, @@ -556,8 +536,7 @@ class IMDbBase: object when accessing its text fields (none, so far).""" companyID = self._normalize_companyID(companyID) companyID = self._get_real_companyID(companyID) - company = Company.Company(companyID=companyID, - accessSystem=self.accessSystem) + company = Company.Company(companyID=companyID, accessSystem=self.accessSystem) modFunct = modFunct or self._defModFunct if modFunct is not None: company.set_mod_funct(modFunct) @@ -580,8 +559,6 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 20 - if not isinstance(name, unicode): - name = unicode(name, encoding, 'replace') res = self._search_company(name, results) return [Company.Company(companyID=self._get_real_companyID(pi), data=pd, modFunct=self._defModFunct, @@ -601,8 +578,6 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 100 - if not isinstance(keyword, unicode): - keyword = unicode(keyword, encoding, 'replace') return self._search_keyword(keyword, results) def _get_keyword(self, keyword, results): @@ -619,10 +594,6 @@ class IMDbBase: results = int(results) except (ValueError, OverflowError): results = 100 - # XXX: I suppose it will be much safer if the user provides - # an unicode string... this is just a guess. - if not isinstance(keyword, unicode): - keyword = unicode(keyword, encoding, 'replace') res = self._get_keyword(keyword, results) return [Movie.Movie(movieID=self._get_real_movieID(mi), data=md, modFunct=self._defModFunct, @@ -653,54 +624,22 @@ class IMDbBase: def new_movie(self, *arguments, **keywords): """Return a Movie object.""" # XXX: not really useful... - if 'title' in keywords: - if not isinstance(keywords['title'], unicode): - keywords['title'] = unicode(keywords['title'], - encoding, 'replace') - elif len(arguments) > 1: - if not isinstance(arguments[1], unicode): - arguments[1] = unicode(arguments[1], encoding, 'replace') - return Movie.Movie(accessSystem=self.accessSystem, - *arguments, **keywords) + return Movie.Movie(accessSystem=self.accessSystem, *arguments, **keywords) def new_person(self, *arguments, **keywords): """Return a Person object.""" # XXX: not really useful... - if 'name' in keywords: - if not isinstance(keywords['name'], unicode): - keywords['name'] = unicode(keywords['name'], - encoding, 'replace') - elif len(arguments) > 1: - if not isinstance(arguments[1], unicode): - arguments[1] = unicode(arguments[1], encoding, 'replace') - return Person.Person(accessSystem=self.accessSystem, - *arguments, **keywords) + return Person.Person(accessSystem=self.accessSystem, *arguments, **keywords) def new_character(self, *arguments, **keywords): """Return a Character object.""" # XXX: not really useful... - if 'name' in keywords: - if not isinstance(keywords['name'], unicode): - keywords['name'] = unicode(keywords['name'], - encoding, 'replace') - elif len(arguments) > 1: - if not isinstance(arguments[1], unicode): - arguments[1] = unicode(arguments[1], encoding, 'replace') - return Character.Character(accessSystem=self.accessSystem, - *arguments, **keywords) + return Character.Character(accessSystem=self.accessSystem, *arguments, **keywords) def new_company(self, *arguments, **keywords): """Return a Company object.""" # XXX: not really useful... - if 'name' in keywords: - if not isinstance(keywords['name'], unicode): - keywords['name'] = unicode(keywords['name'], - encoding, 'replace') - elif len(arguments) > 1: - if not isinstance(arguments[1], unicode): - arguments[1] = unicode(arguments[1], encoding, 'replace') - return Company.Company(accessSystem=self.accessSystem, - *arguments, **keywords) + return Company.Company(accessSystem=self.accessSystem, *arguments, **keywords) def update(self, mop, info=None, override=0): """Given a Movie, Person, Character or Company object with only @@ -727,16 +666,15 @@ class IMDbBase: mopID = mop.companyID prefix = 'company' else: - raise IMDbError('object ' + repr(mop) + \ - ' is not a Movie, Person, Character or Company instance') + raise IMDbError('object ' + repr(mop) + + ' is not a Movie, Person, Character or Company instance') if mopID is None: # XXX: enough? It's obvious that there are Characters # objects without characterID, so I think they should # just do nothing, when an i.update(character) is tried. if prefix == 'character': return - raise IMDbDataAccessError( \ - 'the supplied object has null movieID, personID or companyID') + raise IMDbDataAccessError('supplied object has null movieID, personID or companyID') if mop.accessSystem == self.accessSystem: aSystem = self else: @@ -760,21 +698,21 @@ class IMDbBase: continue if not i: continue - self._imdb_logger.debug('retrieving "%s" info set', i) + _imdb_logger.debug('retrieving "%s" info set', i) try: - method = getattr(aSystem, 'get_%s_%s' % - (prefix, i.replace(' ', '_'))) + method = getattr(aSystem, 'get_%s_%s' % (prefix, i.replace(' ', '_'))) except AttributeError: - self._imdb_logger.error('unknown information set "%s"', i) + _imdb_logger.error('unknown information set "%s"', i) # Keeps going. method = lambda *x: {} try: ret = method(mopID) - except Exception, e: - self._imdb_logger.critical('caught an exception retrieving ' \ - 'or parsing "%s" info set for mopID ' \ - '"%s" (accessSystem: %s)', - i, mopID, mop.accessSystem, exc_info=True) + except Exception: + _imdb_logger.critical( + 'caught an exception retrieving or parsing "%s" info set' + ' for mopID "%s" (accessSystem: %s)', + i, mopID, mop.accessSystem, exc_info=True + ) ret = {} # If requested by the user, reraise the exception. if self._reraise_exceptions: @@ -783,7 +721,7 @@ class IMDbBase: if 'data' in ret: res.update(ret['data']) if isinstance(ret['data'], dict): - keys = ret['data'].keys() + keys = list(ret['data'].keys()) if 'info sets' in ret: for ri in ret['info sets']: mop.add_to_current_info(ri, keys, mainInfoset=i) @@ -826,13 +764,11 @@ class IMDbBase: raise NotImplementedError('override this method') def _searchIMDb(self, kind, ton, title_kind=None): - """Search the IMDb akas server for the given title or name.""" - # The Exact Primary search system has gone AWOL, so we resort - # to the mobile search. :-/ + """Search the IMDb www server for the given title or name.""" if not ton: return None ton = ton.strip('"') - aSystem = IMDb('mobile') + aSystem = IMDb() if kind == 'tt': searchFunct = aSystem.search_movie check = 'long imdb title' @@ -916,8 +852,8 @@ class IMDbBase: imdbID = aSystem.get_imdbMovieID(mop.movieID) else: imdbID = aSystem.title2imdbID(build_title(mop, canonical=0, - ptdf=0, appendKind=False), - mop['kind']) + ptdf=0, appendKind=False), + mop['kind']) elif isinstance(mop, Person.Person): if mop.personID is not None: imdbID = aSystem.get_imdbPersonID(mop.personID) @@ -935,8 +871,8 @@ class IMDbBase: else: imdbID = aSystem.company2imdbID(build_company_name(mop)) else: - raise IMDbError('object ' + repr(mop) + \ - ' is not a Movie, Person or Character instance') + raise IMDbError('object ' + repr(mop) + + ' is not a Movie, Person or Character instance') return imdbID def get_imdbURL(self, mop): @@ -954,8 +890,8 @@ class IMDbBase: elif isinstance(mop, Company.Company): url_firstPart = imdbURL_company_main else: - raise IMDbError('object ' + repr(mop) + \ - ' is not a Movie, Person, Character or Company instance') + raise IMDbError('object ' + repr(mop) + + ' is not a Movie, Person, Character or Company instance') return url_firstPart % imdbID def get_special_methods(self): @@ -964,7 +900,7 @@ class IMDbBase: base_methods = [] for name in dir(IMDbBase): member = getattr(IMDbBase, name) - if isinstance(member, MethodType): + if isinstance(member, (MethodType, FunctionType)): base_methods.append(name) for name in dir(self.__class__): if name.startswith('_') or name in base_methods or \ @@ -974,7 +910,6 @@ class IMDbBase: name.startswith('get_character_'): continue member = getattr(self.__class__, name) - if isinstance(member, MethodType): + if isinstance(member, (MethodType, FunctionType)): sm_dict.update({name: member.__doc__}) return sm_dict - diff -pruN 5.1-1/imdb/linguistics.py 6.6-1/imdb/linguistics.py --- 5.1-1/imdb/linguistics.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/linguistics.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,27 +1,29 @@ -""" -linguistics module (imdb package). +# -*- coding: utf-8 -*- -This module provides functions and data to handle in a smart way -languages and articles (in various languages) at the beginning of movie titles. +# Copyright 2009-2017 Davide Alberani +# 2012 Alberto Malagoli +# 2009 H. Turgut Uyar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -Copyright 2009-2012 Davide Alberani - 2012 Alberto Malagoli - 2009 H. Turgut Uyar - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +This module provides functions and data to handle languages and articles +(in various languages) at the beginning of movie titles in a smart way. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals # List of generic articles used when the language of the title is unknown (or # we don't have information about articles in that language). @@ -46,30 +48,28 @@ Foundation, Inc., 59 Temple Place, Suite # I've left in the list 'i' (1939 vs 2151) and 'uno' (52 vs 56) # I'm not sure what '-al' is, and so I've left it out... # -# Generic list of articles in utf-8 encoding: -GENERIC_ARTICLES = ('the', 'la', 'a', 'die', 'der', 'le', 'el', - "l'", 'il', 'das', 'les', 'i', 'o', 'ein', 'un', 'de', 'los', - 'an', 'una', 'las', 'eine', 'den', 'het', 'gli', 'lo', 'os', - 'ang', 'oi', 'az', 'een', 'ha-', 'det', 'ta', 'al-', - 'mga', "un'", 'uno', 'ett', 'dem', 'egy', 'els', 'eines', - '\xc3\x8f', '\xc3\x87', '\xc3\x94\xc3\xaf', '\xc3\x8f\xc3\xa9') +# Generic list of articles in unicode: +GENERIC_ARTICLES = ( + 'the', 'la', 'a', 'die', 'der', 'le', 'el', "l'", 'il', 'das', 'les', 'i', 'o', 'ein', + 'un', 'de', 'los', 'an', 'una', 'las', 'eine', 'den', 'het', 'gli', 'lo', 'os', 'ang', + 'oi', 'az', 'een', 'ha-', 'det', 'ta', 'al-', 'mga', "un'", 'uno', 'ett', 'dem', 'egy', + 'els', 'eines', 'Ã', 'Ç', 'Ôï', 'Ãé' +) # Lists of articles separated by language. If possible, the list should # be sorted by frequency (not very important, but...) # If you want to add a list of articles for another language, mail it -# it at imdbpy-devel@lists.sourceforge.net; non-ascii articles must be utf-8 -# encoded. +# it at imdbpy-devel@lists.sourceforge.net LANG_ARTICLES = { 'English': ('the', 'a', 'an'), - 'Italian': ('la', 'le', "l'", 'il', 'i', 'un', 'una', 'gli', 'lo', "un'", - 'uno'), - 'Spanish': ('la', 'lo', 'el', 'las', 'un', 'los', 'una', 'al', 'del', - 'unos', 'unas', 'uno'), - 'French': ('le', "l'", 'la', 'les', 'un', 'une', 'des', 'au', 'du', '\xc3\xa0 la', - 'de la', 'aux'), + 'Italian': ('la', 'le', "l'", 'il', 'i', 'un', 'una', 'gli', 'lo', "un'", 'uno'), + 'Spanish': ( + 'la', 'lo', 'el', 'las', 'un', 'los', 'una', 'al', 'del', 'unos', 'unas', 'uno' + ), + 'French': ('le', "l'", 'la', 'les', 'un', 'une', 'des', 'au', 'du', 'à la', 'de la', 'aux'), 'Portuguese': ('a', 'as', 'o', 'os', 'um', 'uns', 'uma', 'umas'), - 'Turkish': (), # Some languages doesn't have articles. + 'Turkish': () # Some languages doesn't have articles. } LANG_ARTICLESget = LANG_ARTICLES.get @@ -78,13 +78,40 @@ LANG_ARTICLESget = LANG_ARTICLES.get # If you want to add an entry for another language or country, mail it at # imdbpy-devel@lists.sourceforge.net . LANG_COUNTRIES = { - 'English': ('Canada', 'Swaziland', 'Ghana', 'St. Lucia', 'Liberia', 'Jamaica', 'Bahamas', 'New Zealand', 'Lesotho', 'Kenya', 'Solomon Islands', 'United States', 'South Africa', 'St. Vincent and the Grenadines', 'Fiji', 'UK', 'Nigeria', 'Australia', 'USA', 'St. Kitts and Nevis', 'Belize', 'Sierra Leone', 'Gambia', 'Namibia', 'Micronesia', 'Kiribati', 'Grenada', 'Antigua and Barbuda', 'Barbados', 'Malta', 'Zimbabwe', 'Ireland', 'Uganda', 'Trinidad and Tobago', 'South Sudan', 'Guyana', 'Botswana', 'United Kingdom', 'Zambia'), + 'English': ( + 'Canada', 'Swaziland', 'Ghana', 'St. Lucia', 'Liberia', 'Jamaica', 'Bahamas', + 'New Zealand', 'Lesotho', 'Kenya', 'Solomon Islands', 'United States', 'South Africa', + 'St. Vincent and the Grenadines', 'Fiji', 'UK', 'Nigeria', 'Australia', 'USA', + 'St. Kitts and Nevis', 'Belize', 'Sierra Leone', 'Gambia', 'Namibia', 'Micronesia', + 'Kiribati', 'Grenada', 'Antigua and Barbuda', 'Barbados', 'Malta', 'Zimbabwe', + 'Ireland', 'Uganda', 'Trinidad and Tobago', 'South Sudan', 'Guyana', 'Botswana', + 'United Kingdom', 'Zambia' + ), 'Italian': ('Italy', 'San Marino', 'Vatican City'), - 'Spanish': ('Spain', 'Mexico', 'Argentina', 'Bolivia', 'Guatemala', 'Uruguay', 'Peru', 'Cuba', 'Dominican Republic', 'Panama', 'Costa Rica', 'Ecuador', 'El Salvador', 'Chile', 'Equatorial Guinea', 'Spain', 'Colombia', 'Nicaragua', 'Venezuela', 'Honduras', 'Paraguay'), - 'French': ('Cameroon', 'Burkina Faso', 'Dominica', 'Gabon', 'Monaco', 'France', "Cote d'Ivoire", 'Benin', 'Togo', 'Central African Republic', 'Mali', 'Niger', 'Congo, Republic of', 'Guinea', 'Congo, Democratic Republic of the', 'Luxembourg', 'Haiti', 'Chad', 'Burundi', 'Madagascar', 'Comoros', 'Senegal'), - 'Portuguese': ('Portugal', 'Brazil', 'Sao Tome and Principe', 'Cape Verde', 'Angola', 'Mozambique', 'Guinea-Bissau'), - 'German': ('Liechtenstein', 'Austria', 'West Germany', 'Switzerland', 'East Germany', 'Germany'), - 'Arabic': ('Saudi Arabia', 'Kuwait', 'Jordan', 'Oman', 'Yemen', 'United Arab Emirates', 'Mauritania', 'Lebanon', 'Bahrain', 'Libya', 'Palestinian State (proposed)', 'Qatar', 'Algeria', 'Morocco', 'Iraq', 'Egypt', 'Djibouti', 'Sudan', 'Syria', 'Tunisia'), + 'Spanish': ( + 'Spain', 'Mexico', 'Argentina', 'Bolivia', 'Guatemala', 'Uruguay', 'Peru', 'Cuba', + 'Dominican Republic', 'Panama', 'Costa Rica', 'Ecuador', 'El Salvador', 'Chile', + 'Equatorial Guinea', 'Spain', 'Colombia', 'Nicaragua', 'Venezuela', 'Honduras', + 'Paraguay' + ), + 'French': ( + 'Cameroon', 'Burkina Faso', 'Dominica', 'Gabon', 'Monaco', 'France', "Cote d'Ivoire", + 'Benin', 'Togo', 'Central African Republic', 'Mali', 'Niger', 'Congo, Republic of', + 'Guinea', 'Congo, Democratic Republic of the', 'Luxembourg', 'Haiti', 'Chad', + 'Burundi', 'Madagascar', 'Comoros', 'Senegal' + ), + 'Portuguese': ( + 'Portugal', 'Brazil', 'Sao Tome and Principe', 'Cape Verde', 'Angola', 'Mozambique', + 'Guinea-Bissau' + ), + 'German': ( + 'Liechtenstein', 'Austria', 'West Germany', 'Switzerland', 'East Germany', 'Germany' + ), + 'Arabic': ( + 'Saudi Arabia', 'Kuwait', 'Jordan', 'Oman', 'Yemen', 'United Arab Emirates', + 'Mauritania', 'Lebanon', 'Bahrain', 'Libya', 'Palestinian State (proposed)', 'Qatar', + 'Algeria', 'Morocco', 'Iraq', 'Egypt', 'Djibouti', 'Sudan', 'Syria', 'Tunisia' + ), 'Turkish': ('Turkey', 'Azerbaijan'), 'Swahili': ('Tanzania',), 'Swedish': ('Sweden',), @@ -154,20 +181,20 @@ for lang in LANG_COUNTRIES: COUNTRY_LANG[country] = lang -def toUnicode(articles): - """Convert a list of articles utf-8 encoded to unicode strings.""" - return tuple([art.decode('utf_8') for art in articles]) +def toUTF8(articles): + """Convert a list of unicode articles to utf-8 encoded strings.""" + return tuple([art.encode('utf8') for art in articles]) def toDicts(articles): - """Given a list of utf-8 encoded articles, build two dictionary (one + """Given a list of unicode encoded articles, build two dictionary (one utf-8 encoded and another one with unicode keys) for faster matches.""" - uArticles = toUnicode(articles) - return dict([(x, x) for x in articles]), dict([(x, x) for x in uArticles]) + utf8Articles = toUTF8(articles) + return dict([(x, x) for x in utf8Articles]), dict([(x, x) for x in articles]) def addTrailingSpace(articles): - """From the given list of utf-8 encoded articles, return two + """From the given list of unicode articles, return two lists (one utf-8 encoded and another one in unicode) where a space is added at the end - if the last char is not ' or -.""" _spArticles = [] @@ -175,8 +202,8 @@ def addTrailingSpace(articles): for article in articles: if article[-1] not in ("'", '-'): article += ' ' - _spArticles.append(article) - _spUnicodeArticles.append(article.decode('utf_8')) + _spArticles.append(article.encode('utf8')) + _spUnicodeArticles.append(article) return _spArticles, _spUnicodeArticles @@ -184,6 +211,7 @@ def addTrailingSpace(articles): _ART_CACHE = {} _SP_ART_CACHE = {} + def articlesDictsForLang(lang): """Return dictionaries of articles specific for the given language, or the default one if the language is not known.""" @@ -202,4 +230,3 @@ def spArticlesForLang(lang): spArticles = addTrailingSpace(LANG_ARTICLESget(lang, GENERIC_ARTICLES)) _SP_ART_CACHE[lang] = spArticles return spArticles - Binary files 5.1-1/imdb/locale/ar/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/ar/LC_MESSAGES/imdbpy.mo differ Binary files 5.1-1/imdb/locale/bg/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/bg/LC_MESSAGES/imdbpy.mo differ Binary files 5.1-1/imdb/locale/de/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/de/LC_MESSAGES/imdbpy.mo differ Binary files 5.1-1/imdb/locale/en/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/en/LC_MESSAGES/imdbpy.mo differ Binary files 5.1-1/imdb/locale/fr/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/fr/LC_MESSAGES/imdbpy.mo differ diff -pruN 5.1-1/imdb/locale/generatepot.py 6.6-1/imdb/locale/generatepot.py --- 5.1-1/imdb/locale/generatepot.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/locale/generatepot.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,32 +1,29 @@ -#!/usr/bin/env python -""" -generatepot.py script. +# Copyright 2009 H. Turgut Uyar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This script generates the imdbpy.pot file, from the DTD. - -Copyright 2009 H. Turgut Uyar - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ import re import sys - from datetime import datetime as dt -DEFAULT_MESSAGES = { } + +DEFAULT_MESSAGES = {} ELEMENT_PATTERN = r""" - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. +# Copyright 2009 H. Turgut Uyar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +This package provides scripts and files for internationalization of IMDbPY. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals import gettext import os Binary files 5.1-1/imdb/locale/__init__.pyc and 6.6-1/imdb/locale/__init__.pyc differ diff -pruN 5.1-1/imdb/locale/msgfmt.py 6.6-1/imdb/locale/msgfmt.py --- 5.1-1/imdb/locale/msgfmt.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/locale/msgfmt.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,5 +1,7 @@ -#!/usr/bin/env python -# -*- coding: iso-8859-1 -*- +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Written by Martin v. Löwis + """Generate binary message catalog from textual translation description. This program converts a textual Uniforum-style message catalog (.po file) into @@ -21,162 +23,194 @@ Options: -V --version Display version information and exit. - -Written by Martin v. Löwis , -refactored / fixed by Thomas Waldmann . """ -import sys, os -import getopt, struct, array +import os +import sys +import ast +import getopt +import struct +import array +from email.parser import HeaderParser -__version__ = "1.3" +__version__ = "1.1" -class SyntaxErrorException(Exception): - """raised when having trouble parsing the po file content""" - pass - -class MsgFmt(object): - """transform .po -> .mo format""" - def __init__(self): - self.messages = {} - - def make_filenames(self, filename, outfile=None): - """Compute .mo name from .po name or language""" - if filename.endswith('.po'): - infile = filename - else: - infile = filename + '.po' - if outfile is None: - outfile = os.path.splitext(infile)[0] + '.mo' - return infile, outfile - - def add(self, id, str, fuzzy): - """Add a non-fuzzy translation to the dictionary.""" - if not fuzzy and str: - self.messages[id] = str - - def read_po(self, lines): - ID = 1 - STR = 2 - section = None - fuzzy = False - line_no = 0 - msgid = msgstr = '' - # Parse the catalog - for line in lines: - line_no += 1 - # If we get a comment line after a msgstr, this is a new entry - if line.startswith('#') and section == STR: - self.add(msgid, msgstr, fuzzy) - section = None - fuzzy = False - # Record a fuzzy mark - if line.startswith('#,') and 'fuzzy' in line: - fuzzy = True - # Skip comments - if line.startswith('#'): - continue - # Now we are in a msgid section, output previous section - if line.startswith('msgid'): - if section == STR: - self.add(msgid, msgstr, fuzzy) - fuzzy = False - section = ID - line = line[5:] - msgid = msgstr = '' - # Now we are in a msgstr section - elif line.startswith('msgstr'): - section = STR - line = line[6:] - # Skip empty lines - line = line.strip() - if not line: - continue - # XXX: Does this always follow Python escape semantics? - line = eval(line) - if section == ID: - msgid += line - elif section == STR: - msgstr += line - else: - raise SyntaxErrorException('Syntax error on line %d, before:\n%s' % (line_no, line)) - # Add last entry - if section == STR: - self.add(msgid, msgstr, fuzzy) - - def generate_mo(self): - """Return the generated output.""" - keys = self.messages.keys() - # the keys are sorted in the .mo file - keys.sort() - offsets = [] - ids = '' - strs = '' - for id in keys: - # For each string, we need size and file offset. Each string is NUL - # terminated; the NUL does not count into the size. - offsets.append((len(ids), len(id), len(strs), len(self.messages[id]))) - ids += id + '\0' - strs += self.messages[id] + '\0' - output = [] - # The header is 7 32-bit unsigned integers. We don't use hash tables, so - # the keys start right after the index tables. - # translated string. - keystart = 7*4 + 16*len(keys) - # and the values start after the keys - valuestart = keystart + len(ids) - koffsets = [] - voffsets = [] - # The string table first has the list of keys, then the list of values. - # Each entry has first the size of the string, then the file offset. - for o1, l1, o2, l2 in offsets: - koffsets += [l1, o1 + keystart] - voffsets += [l2, o2 + valuestart] - offsets = koffsets + voffsets - output.append(struct.pack("Iiiiiii", - 0x950412deL, # Magic - 0, # Version - len(keys), # # of entries - 7*4, # start of key index - 7*4 + len(keys)*8, # start of value index - 0, 0)) # size and offset of hash table - output.append(array.array("i", offsets).tostring()) - output.append(ids) - output.append(strs) - return ''.join(output) +MESSAGES = {} + + + +def usage(code, msg=''): + print(__doc__, file=sys.stderr) + if msg: + print(msg, file=sys.stderr) + sys.exit(code) + +def add(id, str, fuzzy): + "Add a non-fuzzy translation to the dictionary." + global MESSAGES + if not fuzzy and str: + MESSAGES[id] = str + + + +def generate(): + "Return the generated output." + global MESSAGES + # the keys are sorted in the .mo file + keys = sorted(MESSAGES.keys()) + offsets = [] + ids = strs = b'' + for id in keys: + # For each string, we need size and file offset. Each string is NUL + # terminated; the NUL does not count into the size. + offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) + ids += id + b'\0' + strs += MESSAGES[id] + b'\0' + output = '' + # The header is 7 32-bit unsigned integers. We don't use hash tables, so + # the keys start right after the index tables. + # translated string. + keystart = 7*4+16*len(keys) + # and the values start after the keys + valuestart = keystart + len(ids) + koffsets = [] + voffsets = [] + # The string table first has the list of keys, then the list of values. + # Each entry has first the size of the string, then the file offset. + for o1, l1, o2, l2 in offsets: + koffsets += [l1, o1+keystart] + voffsets += [l2, o2+valuestart] + offsets = koffsets + voffsets + output = struct.pack("Iiiiiii", + 0x950412de, # Magic + 0, # Version + len(keys), # # of entries + 7*4, # start of key index + 7*4+len(keys)*8, # start of value index + 0, 0) # size and offset of hash table + output += array.array("i", offsets).tostring() + output += ids + output += strs + return output + + + def make(filename, outfile): - mf = MsgFmt() - infile, outfile = mf.make_filenames(filename, outfile) + ID = 1 + STR = 2 + + # Compute .mo name from .po name and arguments + if filename.endswith('.po'): + infile = filename + else: + infile = filename + '.po' + if outfile is None: + outfile = os.path.splitext(infile)[0] + '.mo' + try: - lines = file(infile).readlines() - except IOError, msg: - print >> sys.stderr, msg + lines = open(infile, 'rb').readlines() + except IOError as msg: + print(msg, file=sys.stderr) sys.exit(1) - try: - mf.read_po(lines) - output = mf.generate_mo() - except SyntaxErrorException, msg: - print >> sys.stderr, msg - try: - open(outfile, "wb").write(output) - except IOError, msg: - print >> sys.stderr, msg + section = None + fuzzy = 0 + # Start off assuming Latin-1, so everything decodes without failure, + # until we know the exact encoding + encoding = 'latin-1' + + # Parse the catalog + lno = 0 + for l in lines: + l = l.decode(encoding) + lno += 1 + # If we get a comment line after a msgstr, this is a new entry + if l[0] == '#' and section == STR: + add(msgid, msgstr, fuzzy) + section = None + fuzzy = 0 + # Record a fuzzy mark + if l[:2] == '#,' and 'fuzzy' in l: + fuzzy = 1 + # Skip comments + if l[0] == '#': + continue + # Now we are in a msgid section, output previous section + if l.startswith('msgid') and not l.startswith('msgid_plural'): + if section == STR: + add(msgid, msgstr, fuzzy) + if not msgid: + # See whether there is an encoding declaration + p = HeaderParser() + charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() + if charset: + encoding = charset + section = ID + l = l[5:] + msgid = msgstr = b'' + is_plural = False + # This is a message with plural forms + elif l.startswith('msgid_plural'): + if section != ID: + print('msgid_plural not preceded by msgid on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l[12:] + msgid += b'\0' # separator of singular and plural + is_plural = True + # Now we are in a msgstr section + elif l.startswith('msgstr'): + section = STR + if l.startswith('msgstr['): + if not is_plural: + print('plural without msgid_plural on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l.split(']', 1)[1] + if msgstr: + msgstr += b'\0' # Separator of the various plural forms + else: + if is_plural: + print('indexed msgstr required for plural on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l[6:] + # Skip empty lines + l = l.strip() + if not l: + continue + l = ast.literal_eval(l) + if section == ID: + msgid += l.encode(encoding) + elif section == STR: + msgstr += l.encode(encoding) + else: + print('Syntax error on %s:%d' % (infile, lno), \ + 'before:', file=sys.stderr) + print(l, file=sys.stderr) + sys.exit(1) + # Add last entry + if section == STR: + add(msgid, msgstr, fuzzy) -def usage(code, msg=''): - print >> sys.stderr, __doc__ - if msg: - print >> sys.stderr, msg - sys.exit(code) + # Compute output + output = generate() + try: + open(outfile,"wb").write(output) + except IOError as msg: + print(msg, file=sys.stderr) + + def main(): try: - opts, args = getopt.getopt(sys.argv[1:], 'hVo:', ['help', 'version', 'output-file=']) - except getopt.error, msg: + opts, args = getopt.getopt(sys.argv[1:], 'hVo:', + ['help', 'version', 'output-file=']) + except getopt.error as msg: usage(1, msg) outfile = None @@ -185,14 +219,14 @@ def main(): if opt in ('-h', '--help'): usage(0) elif opt in ('-V', '--version'): - print >> sys.stderr, "msgfmt.py", __version__ + print("msgfmt.py", __version__) sys.exit(0) elif opt in ('-o', '--output-file'): outfile = arg # do it if not args: - print >> sys.stderr, 'No input file given' - print >> sys.stderr, "Try `msgfmt --help' for more information." + print('No input file given', file=sys.stderr) + print("Try `msgfmt --help' for more information.", file=sys.stderr) return for filename in args: @@ -201,4 +235,3 @@ def main(): if __name__ == '__main__': main() - Binary files 5.1-1/imdb/locale/msgfmt.pyc and 6.6-1/imdb/locale/msgfmt.pyc differ Binary files 5.1-1/imdb/locale/pt_BR/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/pt_BR/LC_MESSAGES/imdbpy.mo differ diff -pruN 5.1-1/imdb/locale/rebuildmo.py 6.6-1/imdb/locale/rebuildmo.py --- 5.1-1/imdb/locale/rebuildmo.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/locale/rebuildmo.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,31 +1,28 @@ -#!/usr/bin/env python -""" -rebuildmo.py script. +# Copyright 2009 H. Turgut Uyar +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This script builds the .mo files, from the .po files. - -Copyright 2009 H. Turgut Uyar - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ import glob -import msgfmt import os -#LOCALE_DIR = os.path.dirname(__file__) +import msgfmt + def rebuildmo(): lang_glob = 'imdbpy-*.po' @@ -45,5 +42,4 @@ def rebuildmo(): if __name__ == '__main__': languages = rebuildmo() - print 'Created locale for: %s.' % ' '.join(languages) - + print('Created locale for: %s.' % ' '.join(languages)) Binary files 5.1-1/imdb/locale/rebuildmo.pyc and 6.6-1/imdb/locale/rebuildmo.pyc differ Binary files 5.1-1/imdb/locale/tr/LC_MESSAGES/imdbpy.mo and 6.6-1/imdb/locale/tr/LC_MESSAGES/imdbpy.mo differ diff -pruN 5.1-1/imdb/_logging.py 6.6-1/imdb/_logging.py --- 5.1-1/imdb/_logging.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/_logging.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,63 +1,50 @@ -""" -_logging module (imdb package). +# Copyright 2009-2017 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides the logging facilities used by the imdb package. - -Copyright 2009-2010 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +from __future__ import absolute_import, division, print_function, unicode_literals + import logging -LEVELS = {'debug': logging.DEBUG, - 'info': logging.INFO, - 'warn': logging.WARNING, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL} + +LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARNING, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL +} imdbpyLogger = logging.getLogger('imdbpy') imdbpyStreamHandler = logging.StreamHandler() -imdbpyFormatter = logging.Formatter('%(asctime)s %(levelname)s [%(name)s]' \ - ' %(pathname)s:%(lineno)d: %(message)s') +imdbpyFormatter = logging.Formatter( + '%(asctime)s %(levelname)s [%(name)s] %(pathname)s:%(lineno)d: %(message)s' +) imdbpyStreamHandler.setFormatter(imdbpyFormatter) imdbpyLogger.addHandler(imdbpyStreamHandler) + def setLevel(level): """Set logging level for the main logger.""" level = level.lower().strip() imdbpyLogger.setLevel(LEVELS.get(level, logging.NOTSET)) - imdbpyLogger.log(imdbpyLogger.level, 'set logging threshold to "%s"', - logging.getLevelName(imdbpyLogger.level)) - - -#imdbpyLogger.setLevel(logging.DEBUG) - - -# It can be an idea to have a single function to log and warn: -#import warnings -#def log_and_warn(msg, args=None, logger=None, level=None): -# """Log the message and issue a warning.""" -# if logger is None: -# logger = imdbpyLogger -# if level is None: -# level = logging.WARNING -# if args is None: -# args = () -# #warnings.warn(msg % args, stacklevel=0) -# logger.log(level, msg % args) - + imdbpyLogger.log(logging.INFO, 'set logging threshold to "%s"', + logging.getLevelName(imdbpyLogger.level)) diff -pruN 5.1-1/imdb/Movie.py 6.6-1/imdb/Movie.py --- 5.1-1/imdb/Movie.py 2015-08-28 15:14:31.000000000 +0000 +++ 6.6-1/imdb/Movie.py 2018-08-05 13:36:02.000000000 +0000 @@ -1,38 +1,40 @@ -""" -Movie module (imdb package). +# Copyright 2004-2018 Davide Alberani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +""" This module provides the Movie class, used to store information about a given movie. - -Copyright 2004-2010 Davide Alberani - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ +from __future__ import absolute_import, division, print_function, unicode_literals + from copy import deepcopy from imdb import linguistics -from imdb.utils import analyze_title, build_title, canonicalTitle, \ - flatten, _Container, cmpMovies +from imdb.utils import _Container +from imdb.utils import analyze_title, build_title, canonicalTitle, cmpMovies, flatten class Movie(_Container): """A Movie. - Every information about a movie can be accessed as: + Every information about a movie can be accessed as:: + movieObject['information'] + to get a list of the kind of information stored in a Movie object, use the keys() method; some useful aliases are defined (as "casting" for the "casting director" key); see @@ -43,97 +45,101 @@ class Movie(_Container): # Aliases for some not-so-intuitive keys. keys_alias = { - 'tv schedule': 'airing', - 'user rating': 'rating', - 'plot summary': 'plot', - 'plot summaries': 'plot', - 'directed by': 'director', - 'created by': 'creator', - 'writing credits': 'writer', - 'produced by': 'producer', - 'original music by': 'original music', - 'non-original music by': 'non-original music', - 'music': 'original music', - 'cinematography by': 'cinematographer', - 'cinematography': 'cinematographer', - 'film editing by': 'editor', - 'film editing': 'editor', - 'editing': 'editor', - 'actors': 'cast', - 'actresses': 'cast', - 'casting by': 'casting director', - 'casting': 'casting director', - 'art direction by': 'art direction', - 'set decoration by': 'set decoration', - 'costume design by': 'costume designer', - 'costume design': 'costume designer', - 'makeup department': 'make up', - 'makeup': 'make up', - 'make-up': 'make up', - 'production management': 'production manager', - 'production company': 'production companies', - 'second unit director or assistant director': - 'assistant director', - 'second unit director': 'assistant director', - 'sound department': 'sound crew', - 'costume and wardrobe department': 'costume department', - 'special effects by': 'special effects', - 'visual effects by': 'visual effects', - 'special effects company': 'special effects companies', - 'stunts': 'stunt performer', - 'other crew': 'miscellaneous crew', - 'misc crew': 'miscellaneous crew', - 'miscellaneouscrew': 'miscellaneous crew', - 'crewmembers': 'miscellaneous crew', - 'crew members': 'miscellaneous crew', - 'other companies': 'miscellaneous companies', - 'misc companies': 'miscellaneous companies', - 'miscellaneous company': 'miscellaneous companies', - 'misc company': 'miscellaneous companies', - 'other company': 'miscellaneous companies', - 'aka': 'akas', - 'also known as': 'akas', - 'country': 'countries', - 'production country': 'countries', - 'production countries': 'countries', - 'genre': 'genres', - 'runtime': 'runtimes', - 'lang': 'languages', - 'color': 'color info', - 'cover': 'cover url', - 'full-size cover': 'full-size cover url', - 'seasons': 'number of seasons', - 'language': 'languages', - 'certificate': 'certificates', - 'certifications': 'certificates', - 'certification': 'certificates', - 'miscellaneous links': 'misc links', - 'miscellaneous': 'misc links', - 'soundclips': 'sound clips', - 'videoclips': 'video clips', - 'photographs': 'photo sites', - 'distributor': 'distributors', - 'distribution': 'distributors', - 'distribution companies': 'distributors', - 'distribution company': 'distributors', - 'guest': 'guests', - 'guest appearances': 'guests', - 'tv guests': 'guests', - 'notable tv guest appearances': 'guests', - 'episodes cast': 'guests', - 'episodes number': 'number of episodes', - 'amazon review': 'amazon reviews', - 'merchandising': 'merchandising links', - 'merchandise': 'merchandising links', - 'sales': 'merchandising links', - 'faq': 'faqs', - 'parental guide': 'parents guide', - 'frequently asked questions': 'faqs'} - - keys_tomodify_list = ('plot', 'trivia', 'alternate versions', 'goofs', - 'quotes', 'dvd', 'laserdisc', 'news', 'soundtrack', - 'crazy credits', 'business', 'supplements', - 'video review', 'faqs') + 'tv schedule': 'airing', + 'user rating': 'rating', + 'plot summary': 'plot', + 'plot summaries': 'plot', + 'directed by': 'director', + 'created by': 'creator', + 'writing credits': 'writer', + 'produced by': 'producer', + 'original music by': 'original music', + 'non-original music by': 'non-original music', + 'music': 'original music', + 'cinematography by': 'cinematographer', + 'cinematography': 'cinematographer', + 'film editing by': 'editor', + 'film editing': 'editor', + 'editing': 'editor', + 'actors': 'cast', + 'actresses': 'cast', + 'casting by': 'casting director', + 'casting': 'casting director', + 'art direction by': 'art direction', + 'set decoration by': 'set decoration', + 'costume design by': 'costume designer', + 'costume design': 'costume designer', + 'makeup department': 'make up', + 'makeup': 'make up', + 'make-up': 'make up', + 'production management': 'production manager', + 'production company': 'production companies', + 'second unit director or assistant director': 'assistant director', + 'second unit director': 'assistant director', + 'sound department': 'sound crew', + 'costume and wardrobe department': 'costume department', + 'special effects by': 'special effects', + 'visual effects by': 'visual effects', + 'special effects company': 'special effects companies', + 'stunts': 'stunt performer', + 'other crew': 'miscellaneous crew', + 'misc crew': 'miscellaneous crew', + 'miscellaneouscrew': 'miscellaneous crew', + 'crewmembers': 'miscellaneous crew', + 'crew members': 'miscellaneous crew', + 'other companies': 'miscellaneous companies', + 'misc companies': 'miscellaneous companies', + 'miscellaneous company': 'miscellaneous companies', + 'misc company': 'miscellaneous companies', + 'other company': 'miscellaneous companies', + 'aka': 'akas', + 'also known as': 'akas', + 'country': 'countries', + 'production country': 'countries', + 'production countries': 'countries', + 'genre': 'genres', + 'runtime': 'runtimes', + 'lang': 'languages', + 'color': 'color info', + 'cover': 'cover url', + 'full-size cover': 'full-size cover url', + 'seasons': 'number of seasons', + 'language': 'languages', + 'certificate': 'certificates', + 'certifications': 'certificates', + 'certification': 'certificates', + 'miscellaneous links': 'misc links', + 'miscellaneous': 'misc links', + 'soundclips': 'sound clips', + 'videoclips': 'video clips', + 'photographs': 'photo sites', + 'distributor': 'distributors', + 'distribution': 'distributors', + 'distribution companies': 'distributors', + 'distribution company': 'distributors', + 'guest': 'guests', + 'guest appearances': 'guests', + 'tv guests': 'guests', + 'notable tv guest appearances': 'guests', + 'episodes cast': 'guests', + 'episodes number': 'number of episodes', + 'amazon review': 'amazon reviews', + 'merchandising': 'merchandising links', + 'merchandise': 'merchandising links', + 'sales': 'merchandising links', + 'faq': 'faqs', + 'parental guide': 'parents guide', + 'frequently asked questions': 'faqs' + } + + keys_tomodify_list = ( + 'plot', 'trivia', 'alternate versions', 'goofs', + 'quotes', 'dvd', 'laserdisc', 'news', 'soundtrack', + 'crazy credits', 'business', 'supplements', + 'video review', 'faqs' + ) + + _image_key = 'cover url' cmpFunct = cmpMovies @@ -163,37 +169,36 @@ class Movie(_Container): *modFunct* -- function called returning text fields. """ title = kwds.get('title') - if title and not self.data.has_key('title'): + if title and 'title' not in self.data: self.set_title(title) self.movieID = kwds.get('movieID', None) - self.myTitle = kwds.get('myTitle', u'') + self.myTitle = kwds.get('myTitle', '') def _reset(self): """Reset the Movie object.""" self.movieID = None - self.myTitle = u'' + self.myTitle = '' def set_title(self, title): """Set the title of the movie.""" - # XXX: convert title to unicode, if it's a plain string? d_title = analyze_title(title) self.data.update(d_title) def _additional_keys(self): """Valid keys to append to the data.keys() list.""" addkeys = [] - if self.data.has_key('title'): + if 'title' in self.data: addkeys += ['canonical title', 'long imdb title', 'long imdb canonical title', 'smart canonical title', 'smart long imdb canonical title'] - if self.data.has_key('episode of'): + if 'episode of' in self.data: addkeys += ['long imdb episode title', 'series title', 'canonical series title', 'episode title', 'canonical episode title', 'smart canonical series title', 'smart canonical episode title'] - if self.data.has_key('cover url'): + if 'cover url' in self.data: addkeys += ['full-size cover url'] return addkeys @@ -215,14 +220,14 @@ class Movie(_Container): used) and the language can be forced with the 'lang' argument, otherwise it's auto-detected.""" if title is None: - title = self.data.get('title', u'') + title = self.data.get('title', '') if lang is None: lang = self.guessLanguage() return canonicalTitle(title, lang=lang) def _getitem(self, key): """Handle special keys.""" - if self.data.has_key('episode of'): + if 'episode of' in self.data: if key == 'long imdb episode title': return build_title(self.data) elif key == 'series title': @@ -234,12 +239,12 @@ class Movie(_Container): ser_title = self.data['episode of']['title'] return self.smartCanonicalTitle(ser_title) elif key == 'episode title': - return self.data.get('title', u'') + return self.data.get('title', '') elif key == 'canonical episode title': - return canonicalTitle(self.data.get('title', u'')) + return canonicalTitle(self.data.get('title', '')) elif key == 'smart canonical episode title': - return self.smartCanonicalTitle(self.data.get('title', u'')) - if self.data.has_key('title'): + return self.smartCanonicalTitle(self.data.get('title', '')) + if 'title' in self.data: if key == 'title': return self.data['title'] elif key == 'long imdb title': @@ -249,74 +254,73 @@ class Movie(_Container): elif key == 'smart canonical title': return self.smartCanonicalTitle(self.data['title']) elif key == 'long imdb canonical title': - return build_title(self.data, canonical=1) + return build_title(self.data, canonical=True) elif key == 'smart long imdb canonical title': - return build_title(self.data, canonical=1, - lang=self.guessLanguage()) - if key == 'full-size cover url' and self.data.has_key('cover url'): - return self._re_fullsizeURL.sub('', self.data.get('cover url', '')) + return build_title(self.data, canonical=True, lang=self.guessLanguage()) + if key == 'full-size cover url': + return self.get_fullsizeURL() return None def getID(self): """Return the movieID.""" return self.movieID - def __nonzero__(self): + def __bool__(self): """The Movie is "false" if the self.data does not contain a title.""" # XXX: check the title and the movieID? - if self.data.has_key('title'): return 1 - return 0 + return 'title' in self.data def isSameTitle(self, other): """Return true if this and the compared object have the same long imdb title and/or movieID. """ # XXX: obsolete? - if not isinstance(other, self.__class__): return 0 - if self.data.has_key('title') and \ - other.data.has_key('title') and \ - build_title(self.data, canonical=0) == \ - build_title(other.data, canonical=0): - return 1 + if not isinstance(other, self.__class__): + return False + if 'title' in self.data and 'title' in other.data and \ + build_title(self.data, canonical=False) == build_title(other.data, canonical=False): + return True if self.accessSystem == other.accessSystem and \ self.movieID is not None and self.movieID == other.movieID: - return 1 - return 0 - isSameMovie = isSameTitle # XXX: just for backward compatiblity. + return True + return False + isSameMovie = isSameTitle # XXX: just for backward compatiblity. def __contains__(self, item): """Return true if the given Person object is listed in this Movie, or if the the given Character is represented in this Movie.""" - from Person import Person - from Character import Character - from Company import Company + from .Person import Person + from .Character import Character + from .Company import Company if isinstance(item, Person): - for p in flatten(self.data, yieldDictKeys=1, scalar=Person, - toDescend=(list, dict, tuple, Movie)): + for p in flatten(self.data, yieldDictKeys=True, scalar=Person, + toDescend=(list, dict, tuple, Movie)): if item.isSame(p): - return 1 + return True elif isinstance(item, Character): - for p in flatten(self.data, yieldDictKeys=1, scalar=Person, - toDescend=(list, dict, tuple, Movie)): + for p in flatten(self.data, yieldDictKeys=True, scalar=Person, + toDescend=(list, dict, tuple, Movie)): if item.isSame(p.currentRole): - return 1 + return True elif isinstance(item, Company): - for c in flatten(self.data, yieldDictKeys=1, scalar=Company, - toDescend=(list, dict, tuple, Movie)): + for c in flatten(self.data, yieldDictKeys=True, scalar=Company, + toDescend=(list, dict, tuple, Movie)): if item.isSame(c): - return 1 - return 0 + return True + elif isinstance(item, str): + return item in self.data + return False def __deepcopy__(self, memo): """Return a deep copy of a Movie instance.""" - m = Movie(title=u'', movieID=self.movieID, myTitle=self.myTitle, - myID=self.myID, data=deepcopy(self.data, memo), - currentRole=deepcopy(self.currentRole, memo), - roleIsPerson=self._roleIsPerson, - notes=self.notes, accessSystem=self.accessSystem, - titlesRefs=deepcopy(self.titlesRefs, memo), - namesRefs=deepcopy(self.namesRefs, memo), - charactersRefs=deepcopy(self.charactersRefs, memo)) + m = Movie(title='', movieID=self.movieID, myTitle=self.myTitle, + myID=self.myID, data=deepcopy(self.data, memo), + currentRole=deepcopy(self.currentRole, memo), + roleIsPerson=self._roleIsPerson, + notes=self.notes, accessSystem=self.accessSystem, + titlesRefs=deepcopy(self.titlesRefs, memo), + namesRefs=deepcopy(self.namesRefs, memo), + charactersRefs=deepcopy(self.charactersRefs, memo)) m.current_info = list(self.current_info) m.set_mod_funct(self.modFunct) return m @@ -324,64 +328,60 @@ class Movie(_Container): def __repr__(self): """String representation of a Movie object.""" # XXX: add also currentRole and notes, if present? - if self.has_key('long imdb episode title'): + if 'long imdb episode title' in self: title = self.get('long imdb episode title') else: title = self.get('long imdb title') - r = '' % (self.movieID, self.accessSystem, - title) - if isinstance(r, unicode): r = r.encode('utf_8', 'replace') - return r + return '' % (self.movieID, self.accessSystem, title) def __str__(self): """Simply print the short title.""" - return self.get('title', u'').encode('utf_8', 'replace') - - def __unicode__(self): - """Simply print the short title.""" - return self.get('title', u'') + return self.get('title', '') def summary(self): """Return a string with a pretty-printed summary for the movie.""" - if not self: return u'' - def _nameAndRole(personList, joiner=u', '): + if not self: + return '' + + def _nameAndRole(personList, joiner=', '): """Build a pretty string with name and role.""" nl = [] for person in personList: - n = person.get('name', u'') - if person.currentRole: n += u' (%s)' % person.currentRole + n = person.get('name', '') + if person.currentRole: + n += ' (%s)' % person.currentRole nl.append(n) return joiner.join(nl) - s = u'Movie\n=====\nTitle: %s\n' % \ - self.get('long imdb canonical title', u'') + s = 'Movie\n=====\nTitle: %s\n' % self.get('long imdb canonical title', '') genres = self.get('genres') - if genres: s += u'Genres: %s.\n' % u', '.join(genres) + if genres: + s += 'Genres: %s.\n' % ', '.join(genres) director = self.get('director') if director: - s += u'Director: %s.\n' % _nameAndRole(director) + s += 'Director: %s.\n' % _nameAndRole(director) writer = self.get('writer') if writer: - s += u'Writer: %s.\n' % _nameAndRole(writer) + s += 'Writer: %s.\n' % _nameAndRole(writer) cast = self.get('cast') if cast: cast = cast[:5] - s += u'Cast: %s.\n' % _nameAndRole(cast) + s += 'Cast: %s.\n' % _nameAndRole(cast) runtime = self.get('runtimes') if runtime: - s += u'Runtime: %s.\n' % u', '.join(runtime) + s += 'Runtime: %s.\n' % ', '.join(runtime) countries = self.get('countries') if countries: - s += u'Country: %s.\n' % u', '.join(countries) + s += 'Country: %s.\n' % ', '.join(countries) lang = self.get('languages') if lang: - s += u'Language: %s.\n' % u', '.join(lang) + s += 'Language: %s.\n' % ', '.join(lang) rating = self.get('rating') if rating: - s += u'Rating: %s' % rating + s += 'Rating: %s' % rating nr_votes = self.get('votes') if nr_votes: - s += u' (%s votes)' % nr_votes - s += u'.\n' + s += ' (%s votes)' % nr_votes + s += '.\n' plot = self.get('plot') if not plot: plot = self.get('plot summary') @@ -392,7 +392,5 @@ class Movie(_Container): i = plot.find('::') if i != -1: plot = plot[:i] - s += u'Plot: %s' % plot + s += 'Plot: %s' % plot return s - - diff -pruN 5.1-1/imdb/parser/http/bsouplxml/_bsoup.py 6.6-1/imdb/parser/http/bsouplxml/_bsoup.py --- 5.1-1/imdb/parser/http/bsouplxml/_bsoup.py 2015-08-28 15:14:32.000000000 +0000 +++ 6.6-1/imdb/parser/http/bsouplxml/_bsoup.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,1970 +0,0 @@ -""" -imdb.parser.http._bsoup module (imdb.parser.http package). -This is the BeautifulSoup.py module, not modified; it's included here -so that it's not an external dependency. - -Beautiful Soup -Elixir and Tonic -"The Screen-Scraper's Friend" -http://www.crummy.com/software/BeautifulSoup/ - -Beautiful Soup parses a (possibly invalid) XML or HTML document into a -tree representation. It provides methods and Pythonic idioms that make -it easy to navigate, search, and modify the tree. - -A well-formed XML/HTML document yields a well-formed data -structure. An ill-formed XML/HTML document yields a correspondingly -ill-formed data structure. If your document is only locally -well-formed, you can use this library to find and process the -well-formed part of it. - -Beautiful Soup works with Python 2.2 and up. It has no external -dependencies, but you'll have more success at converting data to UTF-8 -if you also install these three packages: - -* chardet, for auto-detecting character encodings - http://chardet.feedparser.org/ -* cjkcodecs and iconv_codec, which add more encodings to the ones supported - by stock Python. - http://cjkpython.i18n.org/ - -Beautiful Soup defines classes for two main parsing strategies: - - * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific - language that kind of looks like XML. - - * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid - or invalid. This class has web browser-like heuristics for - obtaining a sensible parse tree in the face of common HTML errors. - -Beautiful Soup also defines a class (UnicodeDammit) for autodetecting -the encoding of an HTML or XML document, and converting it to -Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser. - -For more than you ever wanted to know about Beautiful Soup, see the -documentation: -http://www.crummy.com/software/BeautifulSoup/documentation.html - -Here, have some legalese: - -Copyright (c) 2004-2008, Leonard Richardson - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the the Beautiful Soup Consortium and All - Night Kosher Bakery nor the names of its contributors may be - used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. - -""" -from __future__ import generators - -__author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.0.7a" -__copyright__ = "Copyright (c) 2004-2008 Leonard Richardson" -__license__ = "New-style BSD" - -from sgmllib import SGMLParser, SGMLParseError -import codecs -import markupbase -import types -import re -import sgmllib -try: - from htmlentitydefs import name2codepoint -except ImportError: - name2codepoint = {} -try: - set -except NameError: - from sets import Set as set - -#These hacks make Beautiful Soup able to parse XML with namespaces -sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') -markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match - -DEFAULT_OUTPUT_ENCODING = "utf-8" - -# First, the classes that represent markup elements. - -class PageElement: - """Contains the navigational information for some part of the page - (either a tag or a piece of text)""" - - def setup(self, parent=None, previous=None): - """Sets up the initial relations between this element and - other elements.""" - self.parent = parent - self.previous = previous - self.next = None - self.previousSibling = None - self.nextSibling = None - if self.parent and self.parent.contents: - self.previousSibling = self.parent.contents[-1] - self.previousSibling.nextSibling = self - - def replaceWith(self, replaceWith): - oldParent = self.parent - myIndex = self.parent.contents.index(self) - if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: - # We're replacing this element with one of its siblings. - index = self.parent.contents.index(replaceWith) - if index and index < myIndex: - # Furthermore, it comes before this element. That - # means that when we extract it, the index of this - # element will change. - myIndex = myIndex - 1 - self.extract() - oldParent.insert(myIndex, replaceWith) - - def extract(self): - """Destructively rips this element out of the tree.""" - if self.parent: - try: - self.parent.contents.remove(self) - except ValueError: - pass - - #Find the two elements that would be next to each other if - #this element (and any children) hadn't been parsed. Connect - #the two. - lastChild = self._lastRecursiveChild() - nextElement = lastChild.next - - if self.previous: - self.previous.next = nextElement - if nextElement: - nextElement.previous = self.previous - self.previous = None - lastChild.next = None - - self.parent = None - if self.previousSibling: - self.previousSibling.nextSibling = self.nextSibling - if self.nextSibling: - self.nextSibling.previousSibling = self.previousSibling - self.previousSibling = self.nextSibling = None - return self - - def _lastRecursiveChild(self): - "Finds the last element beneath this object to be parsed." - lastChild = self - while hasattr(lastChild, 'contents') and lastChild.contents: - lastChild = lastChild.contents[-1] - return lastChild - - def insert(self, position, newChild): - if (isinstance(newChild, basestring) - or isinstance(newChild, unicode)) \ - and not isinstance(newChild, NavigableString): - newChild = NavigableString(newChild) - - position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent != None: - # We're 'inserting' an element that's already one - # of this object's children. - if newChild.parent == self: - index = self.find(newChild) - if index and index < position: - # Furthermore we're moving it further down the - # list of this object's children. That means that - # when we extract this element, our target index - # will jump down one. - position = position - 1 - newChild.extract() - - newChild.parent = self - previousChild = None - if position == 0: - newChild.previousSibling = None - newChild.previous = self - else: - previousChild = self.contents[position-1] - newChild.previousSibling = previousChild - newChild.previousSibling.nextSibling = newChild - newChild.previous = previousChild._lastRecursiveChild() - if newChild.previous: - newChild.previous.next = newChild - - newChildsLastElement = newChild._lastRecursiveChild() - - if position >= len(self.contents): - newChild.nextSibling = None - - parent = self - parentsNextSibling = None - while not parentsNextSibling: - parentsNextSibling = parent.nextSibling - parent = parent.parent - if not parent: # This is the last element in the document. - break - if parentsNextSibling: - newChildsLastElement.next = parentsNextSibling - else: - newChildsLastElement.next = None - else: - nextChild = self.contents[position] - newChild.nextSibling = nextChild - if newChild.nextSibling: - newChild.nextSibling.previousSibling = newChild - newChildsLastElement.next = nextChild - - if newChildsLastElement.next: - newChildsLastElement.next.previous = newChildsLastElement - self.contents.insert(position, newChild) - - def append(self, tag): - """Appends the given tag to the contents of this tag.""" - self.insert(len(self.contents), tag) - - def findNext(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears after this Tag in the document.""" - return self._findOne(self.findAllNext, name, attrs, text, **kwargs) - - def findAllNext(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.nextGenerator, - **kwargs) - - def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears after this Tag in the document.""" - return self._findOne(self.findNextSiblings, name, attrs, text, - **kwargs) - - def findNextSiblings(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear after this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.nextSiblingGenerator, **kwargs) - fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x - - def findPrevious(self, name=None, attrs={}, text=None, **kwargs): - """Returns the first item that matches the given criteria and - appears before this Tag in the document.""" - return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs) - - def findAllPrevious(self, name=None, attrs={}, text=None, limit=None, - **kwargs): - """Returns all items that match the given criteria and appear - before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.previousGenerator, - **kwargs) - fetchPrevious = findAllPrevious # Compatibility with pre-3.x - - def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): - """Returns the closest sibling to this Tag that matches the - given criteria and appears before this Tag in the document.""" - return self._findOne(self.findPreviousSiblings, name, attrs, text, - **kwargs) - - def findPreviousSiblings(self, name=None, attrs={}, text=None, - limit=None, **kwargs): - """Returns the siblings of this Tag that match the given - criteria and appear before this Tag in the document.""" - return self._findAll(name, attrs, text, limit, - self.previousSiblingGenerator, **kwargs) - fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x - - def findParent(self, name=None, attrs={}, **kwargs): - """Returns the closest parent of this Tag that matches the given - criteria.""" - # NOTE: We can't use _findOne because findParents takes a different - # set of arguments. - r = None - l = self.findParents(name, attrs, 1) - if l: - r = l[0] - return r - - def findParents(self, name=None, attrs={}, limit=None, **kwargs): - """Returns the parents of this Tag that match the given - criteria.""" - - return self._findAll(name, attrs, None, limit, self.parentGenerator, - **kwargs) - fetchParents = findParents # Compatibility with pre-3.x - - #These methods do the real heavy lifting. - - def _findOne(self, method, name, attrs, text, **kwargs): - r = None - l = method(name, attrs, text, 1, **kwargs) - if l: - r = l[0] - return r - - def _findAll(self, name, attrs, text, limit, generator, **kwargs): - "Iterates over a generator looking for things that match." - - if isinstance(name, SoupStrainer): - strainer = name - else: - # Build a SoupStrainer - strainer = SoupStrainer(name, attrs, text, **kwargs) - results = ResultSet(strainer) - g = generator() - while True: - try: - i = g.next() - except StopIteration: - break - if i: - found = strainer.search(i) - if found: - results.append(found) - if limit and len(results) >= limit: - break - return results - - #These Generators can be used to navigate starting from both - #NavigableStrings and Tags. - def nextGenerator(self): - i = self - while i: - i = i.next - yield i - - def nextSiblingGenerator(self): - i = self - while i: - i = i.nextSibling - yield i - - def previousGenerator(self): - i = self - while i: - i = i.previous - yield i - - def previousSiblingGenerator(self): - i = self - while i: - i = i.previousSibling - yield i - - def parentGenerator(self): - i = self - while i: - i = i.parent - yield i - - # Utility methods - def substituteEncoding(self, str, encoding=None): - encoding = encoding or "utf-8" - return str.replace("%SOUP-ENCODING%", encoding) - - def toEncoding(self, s, encoding=None): - """Encodes an object to a string in some encoding, or to Unicode. - .""" - if isinstance(s, unicode): - if encoding: - s = s.encode(encoding) - elif isinstance(s, str): - if encoding: - s = s.encode(encoding) - else: - s = unicode(s) - else: - if encoding: - s = self.toEncoding(str(s), encoding) - else: - s = unicode(s) - return s - -class NavigableString(unicode, PageElement): - - def __new__(cls, value): - """Create a new NavigableString. - - When unpickling a NavigableString, this method is called with - the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be - passed in to the superclass's __new__ or the superclass won't know - how to handle non-ASCII characters. - """ - if isinstance(value, unicode): - return unicode.__new__(cls, value) - return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) - - def __getnewargs__(self): - return (NavigableString.__str__(self),) - - def __getattr__(self, attr): - """text.string gives you text. This is for backwards - compatibility for Navigable*String, but for CData* it lets you - get the string without the CData wrapper.""" - if attr == 'string': - return self - else: - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) - - def __unicode__(self): - return str(self).decode(DEFAULT_OUTPUT_ENCODING) - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - if encoding: - return self.encode(encoding) - else: - return self - -class CData(NavigableString): - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class ProcessingInstruction(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - output = self - if "%SOUP-ENCODING%" in output: - output = self.substituteEncoding(output, encoding) - return "" % self.toEncoding(output, encoding) - -class Comment(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Declaration(NavigableString): - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) - -class Tag(PageElement): - - """Represents a found HTML tag with its attributes and contents.""" - - def _invert(h): - "Cheap function to invert a hash." - i = {} - for k,v in h.items(): - i[v] = k - return i - - XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", - "quot" : '"', - "amp" : "&", - "lt" : "<", - "gt" : ">" } - - XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) - - def _convertEntities(self, match): - """Used in a call to re.sub to replace HTML, XML, and numeric - entities with the appropriate Unicode characters. If HTML - entities are being converted, any unrecognized entities are - escaped.""" - x = match.group(1) - if self.convertHTMLEntities and x in name2codepoint: - return unichr(name2codepoint[x]) - elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: - if self.convertXMLEntities: - return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] - else: - return u'&%s;' % x - elif len(x) > 0 and x[0] == '#': - # Handle numeric entities - if len(x) > 1 and x[1] == 'x': - return unichr(int(x[2:], 16)) - else: - return unichr(int(x[1:])) - - elif self.escapeUnrecognizedEntities: - return u'&%s;' % x - else: - return u'&%s;' % x - - def __init__(self, parser, name, attrs=None, parent=None, - previous=None): - "Basic constructor." - - # We don't actually store the parser object: that lets extracted - # chunks be garbage-collected - self.parserClass = parser.__class__ - self.isSelfClosing = parser.isSelfClosingTag(name) - self.name = name - if attrs == None: - attrs = [] - self.attrs = attrs - self.contents = [] - self.setup(parent, previous) - self.hidden = False - self.containsSubstitutions = False - self.convertHTMLEntities = parser.convertHTMLEntities - self.convertXMLEntities = parser.convertXMLEntities - self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities - - # Convert any HTML, XML, or numeric entities in the attribute values. - convert = lambda(k, val): (k, - re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", - self._convertEntities, - val)) - self.attrs = map(convert, self.attrs) - - def get(self, key, default=None): - """Returns the value of the 'key' attribute for the tag, or - the value given for 'default' if it doesn't have that - attribute.""" - return self._getAttrMap().get(key, default) - - def has_key(self, key): - return self._getAttrMap().has_key(key) - - def __getitem__(self, key): - """tag[key] returns the value of the 'key' attribute for the tag, - and throws an exception if it's not there.""" - return self._getAttrMap()[key] - - def __iter__(self): - "Iterating over a tag iterates over its contents." - return iter(self.contents) - - def __len__(self): - "The length of a tag is the length of its list of contents." - return len(self.contents) - - def __contains__(self, x): - return x in self.contents - - def __nonzero__(self): - "A tag is non-None even if it has no contents." - return True - - def __setitem__(self, key, value): - """Setting tag[key] sets the value of the 'key' attribute for the - tag.""" - self._getAttrMap() - self.attrMap[key] = value - found = False - for i in range(0, len(self.attrs)): - if self.attrs[i][0] == key: - self.attrs[i] = (key, value) - found = True - if not found: - self.attrs.append((key, value)) - self._getAttrMap()[key] = value - - def __delitem__(self, key): - "Deleting tag[key] deletes all 'key' attributes for the tag." - for item in self.attrs: - if item[0] == key: - self.attrs.remove(item) - #We don't break because bad HTML can define the same - #attribute multiple times. - self._getAttrMap() - if self.attrMap.has_key(key): - del self.attrMap[key] - - def __call__(self, *args, **kwargs): - """Calling a tag like a function is the same as calling its - findAll() method. Eg. tag('a') returns a list of all the A tags - found within this tag.""" - return apply(self.findAll, args, kwargs) - - def __getattr__(self, tag): - #print "Getattr %s.%s" % (self.__class__, tag) - if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3: - return self.find(tag[:-3]) - elif tag.find('__') != 0: - return self.find(tag) - raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) - - def __eq__(self, other): - """Returns true iff this tag has the same name, the same attributes, - and the same contents (recursively) as the given tag. - - NOTE: right now this will return false if two tags have the - same attributes in a different order. Should this be fixed?""" - if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): - return False - for i in range(0, len(self.contents)): - if self.contents[i] != other.contents[i]: - return False - return True - - def __ne__(self, other): - """Returns true iff this tag is not identical to the other tag, - as defined in __eq__.""" - return not self == other - - def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): - """Renders this tag as a string.""" - return self.__str__(encoding) - - def __unicode__(self): - return self.__str__(None) - - BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" - + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" - + ")") - - def _sub_entity(self, x): - """Used with a regular expression to substitute the - appropriate XML entity for an XML special character.""" - return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" - - def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Returns a string or Unicode representation of this tag and - its contents. To get Unicode, pass None for encoding. - - NOTE: since Python's HTML parser consumes whitespace, this - method is not certain to reproduce the whitespace present in - the original string.""" - - encodedName = self.toEncoding(self.name, encoding) - - attrs = [] - if self.attrs: - for key, val in self.attrs: - fmt = '%s="%s"' - if isString(val): - if self.containsSubstitutions and '%SOUP-ENCODING%' in val: - val = self.substituteEncoding(val, encoding) - - # The attribute value either: - # - # * Contains no embedded double quotes or single quotes. - # No problem: we enclose it in double quotes. - # * Contains embedded single quotes. No problem: - # double quotes work here too. - # * Contains embedded double quotes. No problem: - # we enclose it in single quotes. - # * Embeds both single _and_ double quotes. This - # can't happen naturally, but it can happen if - # you modify an attribute value after parsing - # the document. Now we have a bit of a - # problem. We solve it by enclosing the - # attribute in single quotes, and escaping any - # embedded single quotes to XML entities. - if '"' in val: - fmt = "%s='%s'" - if "'" in val: - # TODO: replace with apos when - # appropriate. - val = val.replace("'", "&squot;") - - # Now we're okay w/r/t quotes. But the attribute - # value might also contain angle brackets, or - # ampersands that aren't part of entities. We need - # to escape those to XML entities too. - val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) - - attrs.append(fmt % (self.toEncoding(key, encoding), - self.toEncoding(val, encoding))) - close = '' - closeTag = '' - if self.isSelfClosing: - close = ' /' - else: - closeTag = '' % encodedName - - indentTag, indentContents = 0, 0 - if prettyPrint: - indentTag = indentLevel - space = (' ' * (indentTag-1)) - indentContents = indentTag + 1 - contents = self.renderContents(encoding, prettyPrint, indentContents) - if self.hidden: - s = contents - else: - s = [] - attributeString = '' - if attrs: - attributeString = ' ' + ' '.join(attrs) - if prettyPrint: - s.append(space) - s.append('<%s%s%s>' % (encodedName, attributeString, close)) - if prettyPrint: - s.append("\n") - s.append(contents) - if prettyPrint and contents and contents[-1] != "\n": - s.append("\n") - if prettyPrint and closeTag: - s.append(space) - s.append(closeTag) - if prettyPrint and closeTag and self.nextSibling: - s.append("\n") - s = ''.join(s) - return s - - def decompose(self): - """Recursively destroys the contents of this tree.""" - contents = [i for i in self.contents] - for i in contents: - if isinstance(i, Tag): - i.decompose() - else: - i.extract() - self.extract() - - def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): - return self.__str__(encoding, True) - - def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, - prettyPrint=False, indentLevel=0): - """Renders the contents of this tag as a string in the given - encoding. If encoding is None, returns a Unicode string..""" - s=[] - for c in self: - text = None - if isinstance(c, NavigableString): - text = c.__str__(encoding) - elif isinstance(c, Tag): - s.append(c.__str__(encoding, prettyPrint, indentLevel)) - if text and prettyPrint: - text = text.strip() - if text: - if prettyPrint: - s.append(" " * (indentLevel-1)) - s.append(text) - if prettyPrint: - s.append("\n") - return ''.join(s) - - #Soup methods - - def find(self, name=None, attrs={}, recursive=True, text=None, - **kwargs): - """Return only the first child of this Tag matching the given - criteria.""" - r = None - l = self.findAll(name, attrs, recursive, text, 1, **kwargs) - if l: - r = l[0] - return r - findChild = find - - def findAll(self, name=None, attrs={}, recursive=True, text=None, - limit=None, **kwargs): - """Extracts a list of Tag objects that match the given - criteria. You can specify the name of the Tag and any - attributes you want the Tag to have. - - The value of a key-value pair in the 'attrs' map can be a - string, a list of strings, a regular expression object, or a - callable that takes a string and returns whether or not the - string matches for some custom definition of 'matches'. The - same is true of the tag name.""" - generator = self.recursiveChildGenerator - if not recursive: - generator = self.childGenerator - return self._findAll(name, attrs, text, limit, generator, **kwargs) - findChildren = findAll - - # Pre-3.x compatibility methods - first = find - fetch = findAll - - def fetchText(self, text=None, recursive=True, limit=None): - return self.findAll(text=text, recursive=recursive, limit=limit) - - def firstText(self, text=None, recursive=True): - return self.find(text=text, recursive=recursive) - - #Private methods - - def _getAttrMap(self): - """Initializes a map representation of this tag's attributes, - if not already initialized.""" - if not getattr(self, 'attrMap'): - self.attrMap = {} - for (key, value) in self.attrs: - self.attrMap[key] = value - return self.attrMap - - #Generator methods - def childGenerator(self): - for i in range(0, len(self.contents)): - yield self.contents[i] - raise StopIteration - - def recursiveChildGenerator(self): - stack = [(self, 0)] - while stack: - tag, start = stack.pop() - if isinstance(tag, Tag): - for i in range(start, len(tag.contents)): - a = tag.contents[i] - yield a - if isinstance(a, Tag) and tag.contents: - if i < len(tag.contents) - 1: - stack.append((tag, i+1)) - stack.append((a, 0)) - break - raise StopIteration - -# Next, a couple classes to represent queries and their results. -class SoupStrainer: - """Encapsulates a number of ways of matching a markup element (tag or - text).""" - - def __init__(self, name=None, attrs={}, text=None, **kwargs): - self.name = name - if isString(attrs): - kwargs['class'] = attrs - attrs = None - if kwargs: - if attrs: - attrs = attrs.copy() - attrs.update(kwargs) - else: - attrs = kwargs - self.attrs = attrs - self.text = text - - def __str__(self): - if self.text: - return self.text - else: - return "%s|%s" % (self.name, self.attrs) - - def searchTag(self, markupName=None, markupAttrs={}): - found = None - markup = None - if isinstance(markupName, Tag): - markup = markupName - markupAttrs = markup - callFunctionWithTagData = callable(self.name) \ - and not isinstance(markupName, Tag) - - if (not self.name) \ - or callFunctionWithTagData \ - or (markup and self._matches(markup, self.name)) \ - or (not markup and self._matches(markupName, self.name)): - if callFunctionWithTagData: - match = self.name(markupName, markupAttrs) - else: - match = True - markupAttrMap = None - for attr, matchAgainst in self.attrs.items(): - if not markupAttrMap: - if hasattr(markupAttrs, 'get'): - markupAttrMap = markupAttrs - else: - markupAttrMap = {} - for k,v in markupAttrs: - markupAttrMap[k] = v - attrValue = markupAttrMap.get(attr) - if not self._matches(attrValue, matchAgainst): - match = False - break - if match: - if markup: - found = markup - else: - found = markupName - return found - - def search(self, markup): - #print 'looking for %s in %s' % (self, markup) - found = None - # If given a list of items, scan it for a text element that - # matches. - if isList(markup) and not isinstance(markup, Tag): - for element in markup: - if isinstance(element, NavigableString) \ - and self.search(element): - found = element - break - # If it's a Tag, make sure its name or attributes match. - # Don't bother with Tags if we're searching for text. - elif isinstance(markup, Tag): - if not self.text: - found = self.searchTag(markup) - # If it's text, make sure the text matches. - elif isinstance(markup, NavigableString) or \ - isString(markup): - if self._matches(markup, self.text): - found = markup - else: - raise Exception, "I don't know how to match against a %s" \ - % markup.__class__ - return found - - def _matches(self, markup, matchAgainst): - #print "Matching %s against %s" % (markup, matchAgainst) - result = False - if matchAgainst == True and type(matchAgainst) == types.BooleanType: - result = markup != None - elif callable(matchAgainst): - result = matchAgainst(markup) - else: - #Custom match methods take the tag as an argument, but all - #other ways of matching match the tag name as a string. - if isinstance(markup, Tag): - markup = markup.name - if markup and not isString(markup): - markup = unicode(markup) - #Now we know that chunk is either a string, or None. - if hasattr(matchAgainst, 'match'): - # It's a regexp object. - result = markup and matchAgainst.search(markup) - elif isList(matchAgainst): - result = markup in matchAgainst - elif hasattr(matchAgainst, 'items'): - result = markup.has_key(matchAgainst) - elif matchAgainst and isString(markup): - if isinstance(markup, unicode): - matchAgainst = unicode(matchAgainst) - else: - matchAgainst = str(matchAgainst) - - if not result: - result = matchAgainst == markup - return result - -class ResultSet(list): - """A ResultSet is just a list that keeps track of the SoupStrainer - that created it.""" - def __init__(self, source): - list.__init__([]) - self.source = source - -# Now, some helper functions. - -def isList(l): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is listlike.""" - return hasattr(l, '__iter__') \ - or (type(l) in (types.ListType, types.TupleType)) - -def isString(s): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is stringlike.""" - try: - return isinstance(s, unicode) or isinstance(s, basestring) - except NameError: - return isinstance(s, str) - -def buildTagMap(default, *args): - """Turns a list of maps, lists, or scalars into a single map. - Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and - NESTING_RESET_TAGS maps out of lists and partial maps.""" - built = {} - for portion in args: - if hasattr(portion, 'items'): - #It's a map. Merge it. - for k,v in portion.items(): - built[k] = v - elif isList(portion): - #It's a list. Map each item to the default. - for k in portion: - built[k] = default - else: - #It's a scalar. Map it to the default. - built[portion] = default - return built - -# Now, the parser classes. - -class BeautifulStoneSoup(Tag, SGMLParser): - - """This class contains the basic parser and search code. It defines - a parser that knows nothing about tag behavior except for the - following: - - You can't close a tag without closing all the tags it encloses. - That is, "" actually means - "". - - [Another possible explanation is "", but since - this class defines no SELF_CLOSING_TAGS, it will never use that - explanation.] - - This class is useful for parsing XML or made-up markup languages, - or when BeautifulSoup makes an assumption counter to what you were - expecting.""" - - SELF_CLOSING_TAGS = {} - NESTABLE_TAGS = {} - RESET_NESTING_TAGS = {} - QUOTE_TAGS = {} - PRESERVE_WHITESPACE_TAGS = [] - - MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), - lambda x: x.group(1) + ' />'), - (re.compile(']*)>'), - lambda x: '') - ] - - ROOT_TAG_NAME = u'[document]' - - HTML_ENTITIES = "html" - XML_ENTITIES = "xml" - XHTML_ENTITIES = "xhtml" - # TODO: This only exists for backwards-compatibility - ALL_ENTITIES = XHTML_ENTITIES - - # Used when determining whether a text node is all whitespace and - # can be replaced with a single space. A text node that contains - # fancy Unicode spaces (usually non-breaking) should be left - # alone. - STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } - - def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, - markupMassage=True, smartQuotesTo=XML_ENTITIES, - convertEntities=None, selfClosingTags=None, isHTML=False): - """The Soup object is initialized as the 'root tag', and the - provided markup (which can be a string or a file-like object) - is fed into the underlying parser. - - sgmllib will process most bad HTML, and the BeautifulSoup - class has some tricks for dealing with some HTML that kills - sgmllib, but Beautiful Soup can nonetheless choke or lose data - if your data uses self-closing tags or declarations - incorrectly. - - By default, Beautiful Soup uses regexes to sanitize input, - avoiding the vast majority of these problems. If the problems - don't apply to you, pass in False for markupMassage, and - you'll get better performance. - - The default parser massage techniques fix the two most common - instances of invalid HTML that choke sgmllib: - -
(No space between name of closing tag and tag close) - (Extraneous whitespace in declaration) - - You can pass in a custom list of (RE object, replace method) - tuples to get Beautiful Soup to scrub your input the way you - want.""" - - self.parseOnlyThese = parseOnlyThese - self.fromEncoding = fromEncoding - self.smartQuotesTo = smartQuotesTo - self.convertEntities = convertEntities - # Set the rules for how we'll deal with the entities we - # encounter - if self.convertEntities: - # It doesn't make sense to convert encoded characters to - # entities even while you're converting entities to Unicode. - # Just convert it all to Unicode. - self.smartQuotesTo = None - if convertEntities == self.HTML_ENTITIES: - self.convertXMLEntities = False - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = True - elif convertEntities == self.XHTML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = True - self.escapeUnrecognizedEntities = False - elif convertEntities == self.XML_ENTITIES: - self.convertXMLEntities = True - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - else: - self.convertXMLEntities = False - self.convertHTMLEntities = False - self.escapeUnrecognizedEntities = False - - self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) - SGMLParser.__init__(self) - - if hasattr(markup, 'read'): # It's a file-type object. - markup = markup.read() - self.markup = markup - self.markupMassage = markupMassage - try: - self._feed(isHTML=isHTML) - except StopParsing: - pass - self.markup = None # The markup can now be GCed - - def convert_charref(self, name): - """This method fixes a bug in Python's SGMLParser.""" - try: - n = int(name) - except ValueError: - return - if not 0 <= n <= 127 : # ASCII ends at 127, not 255 - return - return self.convert_codepoint(n) - - def _feed(self, inDocumentEncoding=None, isHTML=False): - # Convert the document to Unicode. - markup = self.markup - if isinstance(markup, unicode): - if not hasattr(self, 'originalEncoding'): - self.originalEncoding = None - else: - dammit = UnicodeDammit\ - (markup, [self.fromEncoding, inDocumentEncoding], - smartQuotesTo=self.smartQuotesTo, isHTML=isHTML) - markup = dammit.unicode - self.originalEncoding = dammit.originalEncoding - self.declaredHTMLEncoding = dammit.declaredHTMLEncoding - if markup: - if self.markupMassage: - if not isList(self.markupMassage): - self.markupMassage = self.MARKUP_MASSAGE - for fix, m in self.markupMassage: - markup = fix.sub(m, markup) - # TODO: We get rid of markupMassage so that the - # soup object can be deepcopied later on. Some - # Python installations can't copy regexes. If anyone - # was relying on the existence of markupMassage, this - # might cause problems. - del(self.markupMassage) - self.reset() - - SGMLParser.feed(self, markup) - # Close out any unfinished strings and close all the open tags. - self.endData() - while self.currentTag.name != self.ROOT_TAG_NAME: - self.popTag() - - def __getattr__(self, methodName): - """This method routes method call requests to either the SGMLParser - superclass or the Tag superclass, depending on the method name.""" - #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - - if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ - or methodName.find('do_') == 0: - return SGMLParser.__getattr__(self, methodName) - elif methodName.find('__') != 0: - return Tag.__getattr__(self, methodName) - else: - raise AttributeError - - def isSelfClosingTag(self, name): - """Returns true iff the given string is the name of a - self-closing tag according to this parser.""" - return self.SELF_CLOSING_TAGS.has_key(name) \ - or self.instanceSelfClosingTags.has_key(name) - - def reset(self): - Tag.__init__(self, self, self.ROOT_TAG_NAME) - self.hidden = 1 - SGMLParser.reset(self) - self.currentData = [] - self.currentTag = None - self.tagStack = [] - self.quoteStack = [] - self.pushTag(self) - - def popTag(self): - tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] - - #print "Pop", tag.name - if self.tagStack: - self.currentTag = self.tagStack[-1] - return self.currentTag - - def pushTag(self, tag): - #print "Push", tag.name - if self.currentTag: - self.currentTag.contents.append(tag) - self.tagStack.append(tag) - self.currentTag = self.tagStack[-1] - - def endData(self, containerClass=NavigableString): - if self.currentData: - currentData = u''.join(self.currentData) - if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and - not set([tag.name for tag in self.tagStack]).intersection( - self.PRESERVE_WHITESPACE_TAGS)): - if '\n' in currentData: - currentData = '\n' - else: - currentData = ' ' - self.currentData = [] - if self.parseOnlyThese and len(self.tagStack) <= 1 and \ - (not self.parseOnlyThese.text or \ - not self.parseOnlyThese.search(currentData)): - return - o = containerClass(currentData) - o.setup(self.currentTag, self.previous) - if self.previous: - self.previous.next = o - self.previous = o - self.currentTag.contents.append(o) - - - def _popToTag(self, name, inclusivePop=True): - """Pops the tag stack up to and including the most recent - instance of the given tag. If inclusivePop is false, pops the tag - stack up to but *not* including the most recent instqance of - the given tag.""" - #print "Popping to %s" % name - if name == self.ROOT_TAG_NAME: - return - - numPops = 0 - mostRecentTag = None - for i in range(len(self.tagStack)-1, 0, -1): - if name == self.tagStack[i].name: - numPops = len(self.tagStack)-i - break - if not inclusivePop: - numPops = numPops - 1 - - for i in range(0, numPops): - mostRecentTag = self.popTag() - return mostRecentTag - - def _smartPop(self, name): - - """We need to pop up to the previous tag of this type, unless - one of this tag's nesting reset triggers comes between this - tag and the previous tag of this type, OR unless this tag is a - generic nesting trigger and another generic nesting trigger - comes between this tag and the previous tag of this type. - - Examples: -

FooBar *

* should pop to 'p', not 'b'. -

FooBar *

* should pop to 'table', not 'p'. -

Foo

Bar *

* should pop to 'tr', not 'p'. - -

    • *
    • * should pop to 'ul', not the first 'li'. -
    • ** should pop to 'table', not the first 'tr' - tag should - implicitly close the previous tag within the same
    ** should pop to 'tr', not the first 'td' - """ - - nestingResetTriggers = self.NESTABLE_TAGS.get(name) - isNestable = nestingResetTriggers != None - isResetNesting = self.RESET_NESTING_TAGS.has_key(name) - popTo = None - inclusive = True - for i in range(len(self.tagStack)-1, 0, -1): - p = self.tagStack[i] - if (not p or p.name == name) and not isNestable: - #Non-nestable tags get popped to the top or to their - #last occurance. - popTo = name - break - if (nestingResetTriggers != None - and p.name in nestingResetTriggers) \ - or (nestingResetTriggers == None and isResetNesting - and self.RESET_NESTING_TAGS.has_key(p.name)): - - #If we encounter one of the nesting reset triggers - #peculiar to this tag, or we encounter another tag - #that causes nesting to reset, pop up to but not - #including that tag. - popTo = p.name - inclusive = False - break - p = p.parent - if popTo: - self._popToTag(popTo, inclusive) - - def unknown_starttag(self, name, attrs, selfClosing=0): - #print "Start tag %s: %s" % (name, attrs) - if self.quoteStack: - #This is not a real tag. - #print "<%s> is not real!" % name - attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) - self.handle_data('<%s%s>' % (name, attrs)) - return - self.endData() - - if not self.isSelfClosingTag(name) and not selfClosing: - self._smartPop(name) - - if self.parseOnlyThese and len(self.tagStack) <= 1 \ - and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)): - return - - tag = Tag(self, name, attrs, self.currentTag, self.previous) - if self.previous: - self.previous.next = tag - self.previous = tag - self.pushTag(tag) - if selfClosing or self.isSelfClosingTag(name): - self.popTag() - if name in self.QUOTE_TAGS: - #print "Beginning quote (%s)" % name - self.quoteStack.append(name) - self.literal = 1 - return tag - - def unknown_endtag(self, name): - #print "End tag %s" % name - if self.quoteStack and self.quoteStack[-1] != name: - #This is not a real end tag. - #print " is not real!" % name - self.handle_data('' % name) - return - self.endData() - self._popToTag(name) - if self.quoteStack and self.quoteStack[-1] == name: - self.quoteStack.pop() - self.literal = (len(self.quoteStack) > 0) - - def handle_data(self, data): - self.currentData.append(data) - - def _toStringSubclass(self, text, subclass): - """Adds a certain piece of text to the tree as a NavigableString - subclass.""" - self.endData() - self.handle_data(text) - self.endData(subclass) - - def handle_pi(self, text): - """Handle a processing instruction as a ProcessingInstruction - object, possibly one with a %SOUP-ENCODING% slot into which an - encoding will be plugged later.""" - if text[:3] == "xml": - text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" - self._toStringSubclass(text, ProcessingInstruction) - - def handle_comment(self, text): - "Handle comments as Comment objects." - self._toStringSubclass(text, Comment) - - def handle_charref(self, ref): - "Handle character references as data." - if self.convertEntities: - data = unichr(int(ref)) - else: - data = '&#%s;' % ref - self.handle_data(data) - - def handle_entityref(self, ref): - """Handle entity references as data, possibly converting known - HTML and/or XML entity references to the corresponding Unicode - characters.""" - data = None - if self.convertHTMLEntities: - try: - data = unichr(name2codepoint[ref]) - except KeyError: - pass - - if not data and self.convertXMLEntities: - data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) - - if not data and self.convertHTMLEntities and \ - not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): - # TODO: We've got a problem here. We're told this is - # an entity reference, but it's not an XML entity - # reference or an HTML entity reference. Nonetheless, - # the logical thing to do is to pass it through as an - # unrecognized entity reference. - # - # Except: when the input is "&carol;" this function - # will be called with input "carol". When the input is - # "AT&T", this function will be called with input - # "T". We have no way of knowing whether a semicolon - # was present originally, so we don't know whether - # this is an unknown entity or just a misplaced - # ampersand. - # - # The more common case is a misplaced ampersand, so I - # escape the ampersand and omit the trailing semicolon. - data = "&%s" % ref - if not data: - # This case is different from the one above, because we - # haven't already gone through a supposedly comprehensive - # mapping of entities to Unicode characters. We might not - # have gone through any mapping at all. So the chances are - # very high that this is a real entity, and not a - # misplaced ampersand. - data = "&%s;" % ref - self.handle_data(data) - - def handle_decl(self, data): - "Handle DOCTYPEs and the like as Declaration objects." - self._toStringSubclass(data, Declaration) - - def parse_declaration(self, i): - """Treat a bogus SGML declaration as raw data. Treat a CDATA - declaration as a CData object.""" - j = None - if self.rawdata[i:i+9] == '', i) - if k == -1: - k = len(self.rawdata) - data = self.rawdata[i+9:k] - j = k+3 - self._toStringSubclass(data, CData) - else: - try: - j = SGMLParser.parse_declaration(self, i) - except SGMLParseError: - toHandle = self.rawdata[i:] - self.handle_data(toHandle) - j = i + len(toHandle) - return j - -class BeautifulSoup(BeautifulStoneSoup): - - """This parser knows the following facts about HTML: - - * Some tags have no closing tag and should be interpreted as being - closed as soon as they are encountered. - - * The text inside some tags (ie. 'script') may contain tags which - are not really part of the document and which should be parsed - as text, not tags. If you want to parse the text as tags, you can - always fetch it and parse it explicitly. - - * Tag nesting rules: - - Most tags can't be nested at all. For instance, the occurance of - a

    tag should implicitly close the previous

    tag. - -

    Para1

    Para2 - should be transformed into: -

    Para1

    Para2 - - Some tags can be nested arbitrarily. For instance, the occurance - of a

    tag should _not_ implicitly close the previous -
    tag. - - Alice said:
    Bob said:
    Blah - should NOT be transformed into: - Alice said:
    Bob said:
    Blah - - Some tags can be nested, but the nesting is reset by the - interposition of other tags. For instance, a
    , - but not close a tag in another table. - -
    BlahBlah - should be transformed into: -
    BlahBlah - but, - Blah
    Blah - should NOT be transformed into - Blah
    Blah - - Differing assumptions about tag nesting rules are a major source - of problems with the BeautifulSoup class. If BeautifulSoup is not - treating as nestable a tag your page author treats as nestable, - try ICantBelieveItsBeautifulSoup, MinimalSoup, or - BeautifulStoneSoup before writing your own subclass.""" - - def __init__(self, *args, **kwargs): - if not kwargs.has_key('smartQuotesTo'): - kwargs['smartQuotesTo'] = self.HTML_ENTITIES - kwargs['isHTML'] = True - BeautifulStoneSoup.__init__(self, *args, **kwargs) - - SELF_CLOSING_TAGS = buildTagMap(None, - ['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) - - PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) - - QUOTE_TAGS = {'script' : None, 'textarea' : None} - - #According to the HTML standard, each of these inline tags can - #contain another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center'] - - #According to the HTML standard, these block tags can contain - #another tag of the same type. Furthermore, it's common - #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] - - #Lists can contain other lists, but there are restrictions. - NESTABLE_LIST_TAGS = { 'ol' : [], - 'ul' : [], - 'li' : ['ul', 'ol'], - 'dl' : [], - 'dd' : ['dl'], - 'dt' : ['dl'] } - - #Tables can contain other tables, but there are restrictions. - NESTABLE_TABLE_TAGS = {'table' : [], - 'tr' : ['table', 'tbody', 'tfoot', 'thead'], - 'td' : ['tr'], - 'th' : ['tr'], - 'thead' : ['table'], - 'tbody' : ['table'], - 'tfoot' : ['table'], - } - - NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] - - #If one of these tags is encountered, all tags up to the next tag of - #this type are popped. - RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript', - NON_NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, - NESTABLE_TABLE_TAGS) - - NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS, - NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) - - # Used to detect the charset in a META tag; see start_meta - CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) - - def start_meta(self, attrs): - """Beautiful Soup can detect a charset included in a META tag, - try to convert the document to that charset, and re-parse the - document from the beginning.""" - httpEquiv = None - contentType = None - contentTypeIndex = None - tagNeedsEncodingSubstitution = False - - for i in range(0, len(attrs)): - key, value = attrs[i] - key = key.lower() - if key == 'http-equiv': - httpEquiv = value - elif key == 'content': - contentType = value - contentTypeIndex = i - - if httpEquiv and contentType: # It's an interesting meta tag. - match = self.CHARSET_RE.search(contentType) - if match: - if (self.declaredHTMLEncoding is not None or - self.originalEncoding == self.fromEncoding): - # An HTML encoding was sniffed while converting - # the document to Unicode, or an HTML encoding was - # sniffed during a previous pass through the - # document, or an encoding was specified - # explicitly and it worked. Rewrite the meta tag. - def rewrite(match): - return match.group(1) + "%SOUP-ENCODING%" - newAttr = self.CHARSET_RE.sub(rewrite, contentType) - attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], - newAttr) - tagNeedsEncodingSubstitution = True - else: - # This is our first pass through the document. - # Go through it again with the encoding information. - newCharset = match.group(3) - if newCharset and newCharset != self.originalEncoding: - self.declaredHTMLEncoding = newCharset - self._feed(self.declaredHTMLEncoding) - raise StopParsing - pass - tag = self.unknown_starttag("meta", attrs) - if tag and tagNeedsEncodingSubstitution: - tag.containsSubstitutions = True - -class StopParsing(Exception): - pass - -class ICantBelieveItsBeautifulSoup(BeautifulSoup): - - """The BeautifulSoup class is oriented towards skipping over - common HTML errors like unclosed tags. However, sometimes it makes - errors of its own. For instance, consider this fragment: - - FooBar - - This is perfectly valid (if bizarre) HTML. However, the - BeautifulSoup class will implicitly close the first b tag when it - encounters the second 'b'. It will think the author wrote - "FooBar", and didn't close the first 'b' tag, because - there's no real-world reason to bold something that's already - bold. When it encounters '' it will close two more 'b' - tags, for a grand total of three tags closed instead of two. This - can throw off the rest of your document structure. The same is - true of a number of other tags, listed below. - - It's much more common for someone to forget to close a 'b' tag - than to actually use nested 'b' tags, and the BeautifulSoup class - handles the common case. This class handles the not-co-common - case: where you can't believe someone wrote what they did, but - it's valid HTML and BeautifulSoup screwed up by assuming it - wouldn't be.""" - - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', - 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big'] - - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] - - NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, - I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS) - -class MinimalSoup(BeautifulSoup): - """The MinimalSoup class is for parsing HTML that contains - pathologically bad markup. It makes no assumptions about tag - nesting, but it does know which tags are self-closing, that -