From 818ff8f2933c6593a934bff247a93334277433cf Mon Sep 17 00:00:00 2001 From: Andres Vazquez Date: Sun, 14 Feb 2021 18:40:02 -0300 Subject: [PATCH] Terminal commands added --- .gitignore | 1 + CHANGELOG | 10 ++- README.md | 64 ++++++++++++++++++- setup.py | 7 +- tests/test_real_streams.py | 11 ++++ yaasr/__init__.py | 13 +++- yaasr/log.conf | 14 +++- yaasr/recorder/stream.py | 17 +++-- .../radio-bio-bio-santiago-chile/README.md | 1 + .../data.json | 2 +- yaasr/terminal/__init__.py | 63 ++++++++++++++++++ 11 files changed, 189 insertions(+), 14 deletions(-) create mode 100644 tests/test_real_streams.py create mode 100644 yaasr/streams/radio-bio-bio-santiago-chile/README.md create mode 100644 yaasr/terminal/__init__.py diff --git a/.gitignore b/.gitignore index 35c7717..7fb1799 100644 --- a/.gitignore +++ b/.gitignore @@ -129,6 +129,7 @@ dmypy.json .pyre/ .vscode/ +*.aac *.mp3 *.mp4 *.ogg diff --git a/CHANGELOG b/CHANGELOG index f21b4b2..933db00 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ # Changelog +## [0.3.3] 2021-02-14 + +### Command line tools added + - `yaasr ls` + - `yaasr test` + - `yaasr info` + - `yaasr record` + ## [0.3.2] 2021-02-14 ### Post processing audios @@ -9,8 +17,6 @@ Allow post-process functions: - Allow upload final audios using SSH - Allow delete previous version file after post-process - - ## [0.2.1] 2021-02-13 ### Added diff --git a/README.md b/README.md index 68f8699..91d1bb0 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,26 @@ Install pip install yaasr ``` -Load a pre-defined stream +### From Python + +Load a pre-defined stream and save 5 audio chunks of 60 seconds + +```python +from yaasr.recorder.stream import YStream +from yaasr.processors.audio.reduce import reformat +from yaasr.processors.archive.ssh import upload_ssh + +ys = YStream('radio-universidad-cordoba-argentina') +ys.load() +ys.record(total_seconds=300, chunk_bytes_size=1024, chunk_time_size=60) +``` + +You will see new audio files at `/yaasr/streams/radio-universidad-cordoba-argentina` + +Post process audio to MP3 16Khz and upload via ssh the result cleaning local files after the process ```python +import logging.config from yaasr.recorder.stream import YStream from yaasr.processors.audio.reduce import reformat from yaasr.processors.archive.ssh import upload_ssh @@ -97,3 +114,48 @@ Results ``` ![ssh files](docs/img/sshed.png) + +### From command line + +List all available streams + +``` +$ yaasr ls +radio-bio-bio-santiago-chile: https://unlimited4-us.dps.live/biobiosantiago/aac/icecast.audio +radio-universidad-cordoba-argentina: https://sp4.colombiatelecom.com.co:10995/stream +``` + +Info about a stream + +``` +$ yaasr info --stream radio-bio-bio-santiago-chile +{ + "title": "Bio Bio Santiago de Chile", + "web": "https://vivo.biobiochile.cl/player/", + "streams": [ + { + "url": "https://unlimited4-us.dps.live/biobiosantiago/aac/icecast.audio", + "extension": "aac" + } + ] +} +``` + +Record a stream + +``` +$ yaasr record \ + --stream radio-bio-bio-santiago-chile \ + --total_seconds 90 \ + --chunk_bytes_size 512 \ + --chunk_time_size 30 + +2021-02-14 18:27:20,382 - yaasr.recorder.stream - INFO - Attempt to record from https://unlimited4-us.dps.live/biobiosantiago/aac/icecast.audio +2021-02-14 18:27:21,244 - yaasr.recorder.stream - INFO - Recording from https://unlimited4-us.dps.live/biobiosantiago/aac/icecast.audio +2021-02-14 18:27:56,923 - yaasr.recorder.stream - INFO - 2021-02-14 18:27:56.923239 Elapsed 0:00:35.679521 Finish chunk 1274 +2021-02-14 18:27:56,924 - yaasr.recorder.stream - INFO - Chunk finished +2021-02-14 18:28:27,132 - yaasr.recorder.stream - INFO - 2021-02-14 18:28:27.131768 Elapsed 0:01:05.888050 Finish chunk 1981 +2021-02-14 18:28:27,132 - yaasr.recorder.stream - INFO - Chunk finished +2021-02-14 18:28:51,294 - yaasr.recorder.stream - INFO - Finish recording 2021-02-14 18:28:51.294881 +2021-02-14 18:28:51,295 - yaasr.recorder.stream - INFO - Chunk finished +``` \ No newline at end of file diff --git a/setup.py b/setup.py index 3d17f58..ca8dfae 100644 --- a/setup.py +++ b/setup.py @@ -26,5 +26,10 @@ python_requires='>=3.6', install_requires=[ 'requests>=2.25.1' - ] + ], + entry_points={ + 'console_scripts': [ + 'yaasr=yaasr.terminal:main', + ], + }, ) diff --git a/tests/test_real_streams.py b/tests/test_real_streams.py new file mode 100644 index 0000000..771414b --- /dev/null +++ b/tests/test_real_streams.py @@ -0,0 +1,11 @@ +from yaasr import get_all_streams +from yaasr.recorder.stream import YStream + + +def test_all_streams(): + streams = get_all_streams() + + for stream in streams: + ys = YStream(stream) + ys.load() + # it works ys.record(total_seconds=30, chunk_bytes_size=512, chunk_time_size=10) diff --git a/yaasr/__init__.py b/yaasr/__init__.py index 045ac3e..88ec0b2 100644 --- a/yaasr/__init__.py +++ b/yaasr/__init__.py @@ -1,6 +1,17 @@ import os -__VERSION__ = '0.3.2' +__VERSION__ = '0.3.3' BASE_FOLDER = os.path.dirname(__file__) STREAMS_FOLDER = os.path.join(BASE_FOLDER, "streams") + + +def get_all_streams(): + streams = [] + for subdir, dirs, files in os.walk(STREAMS_FOLDER): + for sdir in dirs: + data_file = os.path.join(STREAMS_FOLDER, sdir, 'data.json') + if os.path.isfile(data_file): + streams.append(sdir) + + return streams diff --git a/yaasr/log.conf b/yaasr/log.conf index 66cc31f..3b08df5 100644 --- a/yaasr/log.conf +++ b/yaasr/log.conf @@ -1,5 +1,5 @@ [loggers] -keys=root +keys=root,yaasr,pydub [handlers] keys=consoleHandler @@ -11,6 +11,18 @@ keys=simpleFormatter level=INFO handlers=consoleHandler +[logger_yaasr] +level=INFO +handlers=consoleHandler +qualname=yaasr +propagate=0 + +[logger_pydub] +level=INFO +handlers=consoleHandler +qualname=pydub +propagate=0 + [handler_consoleHandler] class=StreamHandler level=INFO diff --git a/yaasr/recorder/stream.py b/yaasr/recorder/stream.py index 489981d..6226c96 100644 --- a/yaasr/recorder/stream.py +++ b/yaasr/recorder/stream.py @@ -35,11 +35,11 @@ def load(self): if not os.path.isfile(data_file): raise StreamDataFileNotFoud(f'Data file not found {data_file}') - data = json.load(open(data_file)) + self.data = json.load(open(data_file)) - self.title = data['title'] - self.web_site = data.get('web', None) - self.streams = data['streams'] + self.title = self.data['title'] + self.web_site = self.data.get('web', None) + self.streams = self.data['streams'] def record(self, total_seconds=300, chunk_bytes_size=1024, chunk_time_size=60): """ Record the online stream @@ -68,16 +68,19 @@ def record(self, total_seconds=300, chunk_bytes_size=1024, chunk_time_size=60): f = open(stream_path, 'wb') logger.info(f'Recording from {url}') last_start = start + c = 0 for block in r.iter_content(chunk_bytes_size): + c += 1 f.write(block) logger.debug(' ... chunk saved') now = datetime.now() - if now - start >= timedelta(seconds=total_seconds): - logger.info('Finish recording') + elapsed = now - start + if elapsed >= timedelta(seconds=total_seconds): + logger.info(f'Finish recording {now}') break elif now - last_start >= timedelta(seconds=chunk_time_size): - logger.info('Finish chink size') + logger.info(f'{now} Elapsed {elapsed} Finish chunk {c}') self.chunk_finished(stream_path) last_start = datetime.now() stime = last_start.strftime(self.str_chunk_time_format) diff --git a/yaasr/streams/radio-bio-bio-santiago-chile/README.md b/yaasr/streams/radio-bio-bio-santiago-chile/README.md new file mode 100644 index 0000000..dd3aa29 --- /dev/null +++ b/yaasr/streams/radio-bio-bio-santiago-chile/README.md @@ -0,0 +1 @@ +# Radio Bio Bio Santiago de Chile diff --git a/yaasr/streams/radio-universidad-cordoba-argentina/data.json b/yaasr/streams/radio-universidad-cordoba-argentina/data.json index 7a1056e..c71b064 100644 --- a/yaasr/streams/radio-universidad-cordoba-argentina/data.json +++ b/yaasr/streams/radio-universidad-cordoba-argentina/data.json @@ -2,6 +2,6 @@ "title": "Radio Universidad de Córdoba", "web": "https://www.cba24n.com.ar/radio-universidad-am-580---cordoba", "streams": [ - {"url": "https://sp4.colombiatelecom.com.co:10995/stream", "extension": "mp4"} + {"url": "https://sp4.colombiatelecom.com.co:10995/stream", "extension": "aac"} ] } \ No newline at end of file diff --git a/yaasr/terminal/__init__.py b/yaasr/terminal/__init__.py new file mode 100644 index 0000000..612e151 --- /dev/null +++ b/yaasr/terminal/__init__.py @@ -0,0 +1,63 @@ +""" +Iterate over priority domains and send data to server +""" +import argparse +import json +import logging.config +from yaasr import get_all_streams +from yaasr.recorder.stream import YStream + + +def ls(test_streams=False): + """ List all available streams """ + # list all streams + streams = get_all_streams() + for stream in streams: + ys = YStream(stream) + ys.load() + print(f'{stream}: {ys.streams[0]["url"]}') + if test_streams: + ys.record(total_seconds=30, chunk_bytes_size=512, chunk_time_size=1000) + + +def info(stream): + """ Shows stream info """ + # list all streams + ys = YStream(stream) + ys.load() + data = json.dumps(ys.data, indent=4) + print(data) + + +def record(stream, total_seconds=300, chunk_bytes_size=256, chunk_time_size=60): + """ Shows stream info """ + # list all streams + ys = YStream(stream) + ys.load() + ys.record(total_seconds=total_seconds, chunk_bytes_size=chunk_bytes_size, chunk_time_size=chunk_time_size) + + +def main(): + logging.config.fileConfig('yaasr/log.conf') + parser = argparse.ArgumentParser() + parser.add_argument('command', help='Command to run') + parser.add_argument('--log_level', nargs='?', default='INFO', type=str) + parser.add_argument('--stream', nargs='?', default=None, type=str) + parser.add_argument('--total_seconds', nargs='?', default=300, type=int) + parser.add_argument('--chunk_bytes_size', nargs='?', default=256, type=int) + parser.add_argument('--chunk_time_size', nargs='?', default=60, type=int) + + args = parser.parse_args() + + if args.command == 'ls': + return ls() + elif args.command == 'test': + return ls(test_stream=True) + elif args.command == 'info': + return info(stream=args.stream) + elif args.command == 'record': + return record( + stream=args.stream, + total_seconds=args.total_seconds, + chunk_bytes_size=args.chunk_bytes_size, + chunk_time_size=args.chunk_time_size)