From 635d147a40b8d6fd97e0c4766ee30cbfebb55526 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Fri, 19 Oct 2018 12:10:28 +0530 Subject: [PATCH 01/15] Changes made: - Added new logic to implement sort and search filters for YouTube search filters in YoutubeSearchQueryHandlerFactory. - Corrected some typos in SearchQueryHandlerFactory. - Changed test cases affected by changes made to YoutubeSearchQueryHandlerFactory. - Added todo items for the 2 primary test cases in YoutubeSearchQHTest. --- .../SearchQueryHandlerFactory.java | 10 +- .../YoutubeSearchQueryHandlerFactory.java | 176 +++++++++++++++--- .../search/YoutubeSearchCountTest.java | 4 +- ...YoutubeSearchExtractorChannelOnlyTest.java | 29 ++- .../youtube/search/YoutubeSearchQHTest.java | 50 ++--- gradlew | 2 +- 6 files changed, 209 insertions(+), 62 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java index 997e2d09c0..8f571a7724 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java @@ -12,7 +12,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { /////////////////////////////////// @Override - public abstract String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException; + public abstract String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException; public String getSearchString(String url) { return "";} /////////////////////////////////// @@ -23,14 +23,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory { public String getId(String url) { return getSearchString(url); } @Override - public SearchQueryHandler fromQuery(String querry, + public SearchQueryHandler fromQuery(String query, List contentFilter, String sortFilter) throws ParsingException { - return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter)); + return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter)); } - public SearchQueryHandler fromQuery(String querry) throws ParsingException { - return fromQuery(querry, new ArrayList(0), ""); + public SearchQueryHandler fromQuery(String query) throws ParsingException { + return fromQuery(query, new ArrayList(0), ""); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index ba5ee15203..13d94b3020 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -3,18 +3,40 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.xml.bind.DatatypeConverter; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public static final String CHARSET_UTF_8 = "UTF-8"; - public static final String VIDEOS = "videos"; - public static final String CHANNELS = "channels"; - public static final String PLAYLISTS = "playlists"; - public static final String ALL = "all"; + private static final SortFilter DEFAULT_SORT_FILTER = SortFilter.relevance; + private static final ContentFilter DEFAULT_CONTENT_FILTER = ContentFilter.all; + + public enum ContentFilter { + all, + videos, + channels, + playlists, + movie, + show + } + + public enum SortFilter { + relevance, + rating, + upload_date, + date, + view_count, + views + } public static YoutubeSearchQueryHandlerFactory getInstance() { return new YoutubeSearchQueryHandlerFactory(); @@ -23,20 +45,12 @@ public static YoutubeSearchQueryHandlerFactory getInstance() { @Override public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { try { - final String url = "https://www.youtube.com/results" - + "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8); - - if(contentFilters.size() > 0) { - switch (contentFilters.get(0)) { - case VIDEOS: return url + "&sp=EgIQAVAU"; - case CHANNELS: return url + "&sp=EgIQAlAU"; - case PLAYLISTS: return url + "&sp=EgIQA1AU"; - case ALL: - default: - } + String returnURL = getSearchBaseUrl(searchString); + String filterQueryParams = getFilterQueryParams(contentFilters, sortFilter); + if(filterQueryParams != null) { + returnURL = returnURL + "&sp=" + filterQueryParams; } - - return url; + return returnURL; } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); } @@ -44,10 +58,128 @@ public String getUrl(String searchString, List contentFilters, String so @Override public String[] getAvailableContentFilter() { - return new String[] { - ALL, - VIDEOS, - CHANNELS, - PLAYLISTS}; + List contentFiltersList = new ArrayList<>(); + for(ContentFilter contentFilter : ContentFilter.values()) { + contentFiltersList.add(contentFilter.name()); + } + String[] contentFiltersArray = new String[contentFiltersList.size()]; + contentFiltersArray = contentFiltersList.toArray(contentFiltersArray); + return contentFiltersArray; + } + + @Override + public String[] getAvailableSortFilter() { + List sortFiltersList = new ArrayList<>(); + for(SortFilter sortFilter : SortFilter.values()) { + sortFiltersList.add(sortFilter.name()); + } + String[] sortFiltersArray = new String[sortFiltersList.size()]; + sortFiltersArray = sortFiltersList.toArray(sortFiltersArray); + return sortFiltersArray; + } + + private String getSearchBaseUrl(String searchQuery) + throws UnsupportedEncodingException { + return "https://www.youtube.com/results" + + "?q=" + + URLEncoder.encode(searchQuery, CHARSET_UTF_8); + } + + @Nullable + private String getFilterQueryParams(List contentFilters, String sortFilter) + throws UnsupportedEncodingException { + List returnList = new ArrayList<>(); + List sortFilterParams = getSortFiltersQueryParam(sortFilter); + if(!sortFilterParams.isEmpty()) { + returnList.addAll(sortFilterParams); + } + List contentFilterParams = getContentFiltersQueryParams(contentFilters); + if(!contentFilterParams.isEmpty()) { + returnList.add((byte)0x12); + returnList.add((byte)contentFilterParams.size()); + returnList.addAll(contentFilterParams); + } + + if(returnList.isEmpty()) { + return null; + } + return URLEncoder.encode(DatatypeConverter.printBase64Binary(convert(returnList)), CHARSET_UTF_8); + } + + private List getContentFiltersQueryParams(List contentFilter) { + if(contentFilter == null || contentFilter.isEmpty()) { + return Collections.emptyList(); + } + List returnList = new ArrayList<>(); + for(String filter : contentFilter) { + List byteList = getContentFilterQueryParams(filter); + if(!byteList.isEmpty()) { + returnList.addAll(byteList); + } + } + return returnList; + } + + private List getContentFilterQueryParams(String filter) { + ContentFilter contentFilter; + try { + contentFilter = ContentFilter.valueOf(filter); + } catch (IllegalArgumentException iae) { + iae.printStackTrace(); + System.err.println("Unknown content filter type provided = " + filter +", none will be applied"); + return Collections.emptyList(); + } + switch (contentFilter) { + case all: + return Collections.emptyList(); + case videos: + return Arrays.asList((byte)0x10, (byte)0x01); + case channels: + return Arrays.asList((byte)0x10, (byte)0x02); + case playlists: + return Arrays.asList((byte)0x10, (byte)0x03); + case movie: + return Arrays.asList((byte)0x10, (byte)0x04); + case show: + return Arrays.asList((byte)0x10, (byte)0x05); + default: + throw new IllegalArgumentException("Unexpected content filter found = " + contentFilter); + } + } + + private List getSortFiltersQueryParam(String filter) { + if(filter == null || filter.isEmpty()) { + return Collections.emptyList(); + } + SortFilter sortFilter; + try { + sortFilter = SortFilter.valueOf(filter); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + System.err.println("Unknown sort filter = " + filter + ", provided, none applied."); + return Collections.emptyList(); + } + switch (sortFilter) { + case relevance: + return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x00)); + case rating: + return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x01)); + case upload_date: + case date: + return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x02)); + case view_count: + case views: + return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x03)); + default: + throw new IllegalArgumentException("Unexpected sort filter = " + sortFilter); + } + } + + private byte[] convert(@Nonnull List bigByteList) { + byte[] returnArray = new byte[bigByteList.size()]; + for (int i = 0; i < bigByteList.size(); i++) { + returnArray[i] = bigByteList.get(i); + } + return returnArray; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java index af314e632b..e5996bab8d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ContentFilter; import org.schabi.newpipe.extractor.utils.Localization; import static java.util.Collections.singletonList; @@ -19,7 +19,7 @@ public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBa public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en")); + singletonList(ContentFilter.channels.name()), null, new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 937cee5ecf..df63837ae9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -8,28 +8,35 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ContentFilter; import org.schabi.newpipe.extractor.utils.Localization; -import static java.util.Arrays.asList; +import java.util.Collections; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { + private static final String SEARCH_QUERY = "pewdiepie"; + private static final Localization DEFAULT_LOCALIZATION = new Localization("GB", "en"); + + private static String baseSearchPageURL; + @BeforeClass public static void setUpClass() throws Exception { - NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); - extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en")); + NewPipe.init(Downloader.getInstance(), DEFAULT_LOCALIZATION); + extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor(SEARCH_QUERY, + Collections.singletonList(ContentFilter.channels.name()), null, DEFAULT_LOCALIZATION); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); + baseSearchPageURL = extractor.getUrl(); } @Test public void testGetSecondPage() throws Exception { - YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en")); + YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor(SEARCH_QUERY, + Collections.singletonList(ContentFilter.channels.name()), null, DEFAULT_LOCALIZATION); ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); assertTrue(Integer.toString(secondPage.getItems().size()), secondPage.getItems().size() > 10); @@ -45,12 +52,16 @@ public void testGetSecondPage() throws Exception { } assertFalse("First and second page are equal", equals); - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3&gl=GB", secondPage.getNextPageUrl()); + String thirdSearchPageURL = secondPage.getNextPageUrl(); + assertTrue("Third search page URL contains base search page URL along with page 3 identifier", + thirdSearchPageURL.contains(baseSearchPageURL) && thirdSearchPageURL.contains("&page=3")); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2&gl=GB", extractor.getNextPageUrl()); + String secondSearchPageURL = extractor.getNextPageUrl(); + assertTrue("Second search page URL contains base search page URL along with page 2 identifier", + secondSearchPageURL.contains(baseSearchPageURL) && secondSearchPageURL.contains("&page=2")); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index c3d3f93c4a..3b416dd9eb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -3,12 +3,10 @@ import org.junit.Test; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import static java.util.Arrays.asList; +import java.util.Collections; + import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; -import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; public class YoutubeSearchQHTest { @@ -23,37 +21,43 @@ public void testRegularValues() throws Exception { @Test public void testGetContentFilter() throws Exception { - assertEquals(VIDEOS, YouTube.getSearchQHFactory() - .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); - assertEquals(CHANNELS, YouTube.getSearchQHFactory() - .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); + assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.videos.name(), YouTube.getSearchQHFactory() + .fromQuery("", Collections.singletonList(YoutubeSearchQueryHandlerFactory.ContentFilter.videos.name()), "").getContentFilters().get(0)); + assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.channels.name(), YouTube.getSearchQHFactory() + .fromQuery("asdf", Collections.singletonList(YoutubeSearchQueryHandlerFactory.ContentFilter.channels.name()), "").getContentFilters().get(0)); + } + + @Test + public void testWithGenuineContentfilter() throws Exception { + // TODO: 19/10/18 + } + + @Test + public void testWithGenuineSortfilter() throws Exception { + // TODO: 19/10/18 + } + + @Test + public void testWithGibbershContentFilter() throws Exception { + assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() + .fromQuery("asdf", Collections.singletonList("gibberish"), "").getUrl()); } @Test - public void testWithContentfilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAVAU", YouTube.getSearchQHFactory() - .fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAlAU", YouTube.getSearchQHFactory() - .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQA1AU", YouTube.getSearchQHFactory() - .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); + public void testWithGibbershSortFilter() throws Exception { assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() - .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); + .fromQuery("asdf", Collections.emptyList(), "gibberish").getUrl()); } @Test public void testGetAvailableContentFilter() { final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); - assertEquals(4, contentFilter.length); - assertEquals("all", contentFilter[0]); - assertEquals("videos", contentFilter[1]); - assertEquals("channels", contentFilter[2]); - assertEquals("playlists", contentFilter[3]); + assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.values().length, contentFilter.length); } @Test public void testGetAvailableSortFilter() { - final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableSortFilter(); - assertEquals(0, contentFilter.length); + final String[] sortFilters = YouTube.getSearchQHFactory().getAvailableSortFilter(); + assertEquals(YoutubeSearchQueryHandlerFactory.SortFilter.values().length, sortFilters.length); } } diff --git a/gradlew b/gradlew index cccdd3d517..068a6747d9 100755 --- a/gradlew +++ b/gradlew @@ -30,7 +30,7 @@ APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS="" -# Use the maximum available, or set MAX_FD != -1 to use that value. +# Use the maximum available, or set MAX_FD != -1 to use that values. MAX_FD="maximum" warn () { From d18c68a59f27e6d96aca05ac255bb43616fb8598 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Thu, 8 Nov 2018 21:57:43 +0530 Subject: [PATCH 02/15] Changes made: - Added ability to handle more filters. - Added test cases to appropriately test the new filters implemented in combination with the available sorters. --- .java-version | 1 + .../YoutubeSearchQueryHandlerFactory.java | 117 +++++++----- .../search/YoutubeSearchCountTest.java | 4 +- ...YoutubeSearchExtractorChannelOnlyTest.java | 6 +- .../youtube/search/YoutubeSearchQHTest.java | 171 ++++++++++++++++-- 5 files changed, 227 insertions(+), 72 deletions(-) create mode 100644 .java-version diff --git a/.java-version b/.java-version new file mode 100644 index 0000000000..6259340971 --- /dev/null +++ b/.java-version @@ -0,0 +1 @@ +1.8 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 13d94b3020..5a15ac979e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -17,25 +17,67 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public static final String CHARSET_UTF_8 = "UTF-8"; - private static final SortFilter DEFAULT_SORT_FILTER = SortFilter.relevance; - private static final ContentFilter DEFAULT_CONTENT_FILTER = ContentFilter.all; - - public enum ContentFilter { - all, - videos, - channels, - playlists, - movie, - show + public enum FilterType { + Content((byte)0x10), + Time((byte)0x08), + Duration((byte)0x18); + + private final byte value; + + FilterType(byte value) { + this.value = value; + } + } + + public enum Filter { + All(FilterType.Content,(byte)0), + Video(FilterType.Content,(byte)0x01), + Channel(FilterType.Content,(byte)0x02), + Playlist(FilterType.Content,(byte)0x03), + Movie(FilterType.Content,(byte)0x04), + Show(FilterType.Content,(byte)0x05), + + Hour(FilterType.Time,(byte)0x01), + Today(FilterType.Time,(byte)0x02), + Week(FilterType.Time,(byte)0x03), + Month(FilterType.Time,(byte)0x04), + Year(FilterType.Time,(byte)0x05), + + Short(FilterType.Duration, (byte)0x01), + Long(FilterType.Duration, (byte)0x02); + + private final FilterType type; + private final byte value; + + Filter(FilterType type, byte value) { + this.type = type; + this.value = value; + } + } + + public enum SorterType { + Default((byte)0x08); + + private final byte value; + + SorterType(byte value) { + this.value = value; + } } - public enum SortFilter { - relevance, - rating, - upload_date, - date, - view_count, - views + public enum Sorter { + Relevance(SorterType.Default, (byte)0x00), + Rating(SorterType.Default, (byte)0x01), + Upload_Date(SorterType.Default, (byte)0x02), + View_Count(SorterType.Default, (byte)0x03); + + private final SorterType type; + private final byte value; + + Sorter(SorterType type, byte value) { + this.type = type; + this.value = value; + } } public static YoutubeSearchQueryHandlerFactory getInstance() { @@ -59,7 +101,7 @@ public String getUrl(String searchString, List contentFilters, String so @Override public String[] getAvailableContentFilter() { List contentFiltersList = new ArrayList<>(); - for(ContentFilter contentFilter : ContentFilter.values()) { + for(Filter contentFilter : Filter.values()) { contentFiltersList.add(contentFilter.name()); } String[] contentFiltersArray = new String[contentFiltersList.size()]; @@ -70,7 +112,7 @@ public String[] getAvailableContentFilter() { @Override public String[] getAvailableSortFilter() { List sortFiltersList = new ArrayList<>(); - for(SortFilter sortFilter : SortFilter.values()) { + for(Sorter sortFilter : Sorter.values()) { sortFiltersList.add(sortFilter.name()); } String[] sortFiltersArray = new String[sortFiltersList.size()]; @@ -121,29 +163,19 @@ private List getContentFiltersQueryParams(List contentFilter) { } private List getContentFilterQueryParams(String filter) { - ContentFilter contentFilter; + Filter contentFilter; try { - contentFilter = ContentFilter.valueOf(filter); + contentFilter = Filter.valueOf(filter); } catch (IllegalArgumentException iae) { iae.printStackTrace(); System.err.println("Unknown content filter type provided = " + filter +", none will be applied"); return Collections.emptyList(); } switch (contentFilter) { - case all: + case All: return Collections.emptyList(); - case videos: - return Arrays.asList((byte)0x10, (byte)0x01); - case channels: - return Arrays.asList((byte)0x10, (byte)0x02); - case playlists: - return Arrays.asList((byte)0x10, (byte)0x03); - case movie: - return Arrays.asList((byte)0x10, (byte)0x04); - case show: - return Arrays.asList((byte)0x10, (byte)0x05); default: - throw new IllegalArgumentException("Unexpected content filter found = " + contentFilter); + return Arrays.asList(contentFilter.type.value, contentFilter.value); } } @@ -151,28 +183,15 @@ private List getSortFiltersQueryParam(String filter) { if(filter == null || filter.isEmpty()) { return Collections.emptyList(); } - SortFilter sortFilter; + Sorter sorter; try { - sortFilter = SortFilter.valueOf(filter); + sorter = Sorter.valueOf(filter); } catch (IllegalArgumentException e) { e.printStackTrace(); System.err.println("Unknown sort filter = " + filter + ", provided, none applied."); return Collections.emptyList(); } - switch (sortFilter) { - case relevance: - return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x00)); - case rating: - return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x01)); - case upload_date: - case date: - return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x02)); - case view_count: - case views: - return new ArrayList<>(Arrays.asList((byte)0x08, (byte)0x03)); - default: - throw new IllegalArgumentException("Unexpected sort filter = " + sortFilter); - } + return Arrays.asList(sorter.type.value, sorter.value); } private byte[] convert(@Nonnull List bigByteList) { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java index e5996bab8d..6038124b22 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ContentFilter; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Filter; import org.schabi.newpipe.extractor.utils.Localization; import static java.util.Collections.singletonList; @@ -19,7 +19,7 @@ public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBa public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", - singletonList(ContentFilter.channels.name()), null, new Localization("GB", "en")); + singletonList(Filter.Channel.name()), null, new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index df63837ae9..77925c9bcf 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ContentFilter; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Filter; import org.schabi.newpipe.extractor.utils.Localization; import java.util.Collections; @@ -27,7 +27,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), DEFAULT_LOCALIZATION); extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor(SEARCH_QUERY, - Collections.singletonList(ContentFilter.channels.name()), null, DEFAULT_LOCALIZATION); + Collections.singletonList(Filter.Channel.name()), null, DEFAULT_LOCALIZATION); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); baseSearchPageURL = extractor.getUrl(); @@ -36,7 +36,7 @@ public static void setUpClass() throws Exception { @Test public void testGetSecondPage() throws Exception { YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor(SEARCH_QUERY, - Collections.singletonList(ContentFilter.channels.name()), null, DEFAULT_LOCALIZATION); + Collections.singletonList(Filter.Channel.name()), null, DEFAULT_LOCALIZATION); ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); assertTrue(Integer.toString(secondPage.getItems().size()), secondPage.getItems().size() > 10); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 3b416dd9eb..20f18e9896 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -1,19 +1,41 @@ package org.schabi.newpipe.extractor.services.youtube.search; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.junit.BeforeClass; import org.junit.Test; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Filter; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Sorter; +import org.schabi.newpipe.extractor.utils.Localization; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchQHTest { + private static final String DEFAULT_SEARCH_QUERY = "asdf"; + + @BeforeClass + public static void setupClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + @Test public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?q=" + DEFAULT_SEARCH_QUERY, YouTube.getSearchQHFactory().fromQuery(DEFAULT_SEARCH_QUERY).getUrl()); + assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); @@ -21,43 +43,156 @@ public void testRegularValues() throws Exception { @Test public void testGetContentFilter() throws Exception { - assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.videos.name(), YouTube.getSearchQHFactory() - .fromQuery("", Collections.singletonList(YoutubeSearchQueryHandlerFactory.ContentFilter.videos.name()), "").getContentFilters().get(0)); - assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.channels.name(), YouTube.getSearchQHFactory() - .fromQuery("asdf", Collections.singletonList(YoutubeSearchQueryHandlerFactory.ContentFilter.channels.name()), "").getContentFilters().get(0)); + assertEquals(Filter.Video.name(), YouTube.getSearchQHFactory() + .fromQuery("", Collections.singletonList(Filter.Video.name()), "").getContentFilters().get(0)); + assertEquals(Filter.Channel.name(), YouTube.getSearchQHFactory() + .fromQuery(DEFAULT_SEARCH_QUERY, Collections.singletonList(Filter.Channel.name()), "").getContentFilters().get(0)); + } + + @Test + public void testWithChannelContentFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.singletonList(Filter.Channel.name()), + "" + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element matchingElement = getMatchingElement( + Filter.Channel.name(), + filterList + ); + if(matchingElement == null) { + fail("Channel filter has not been selected"); + } } @Test - public void testWithGenuineContentfilter() throws Exception { - // TODO: 19/10/18 + public void testWithRatingSortFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.emptyList(), + Sorter.Rating.name() + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element matchingElement = getMatchingElement( + Sorter.Rating.name(), + filterList + ); + if(matchingElement == null) { + fail("Rating sorter has not been selected"); + } } @Test - public void testWithGenuineSortfilter() throws Exception { - // TODO: 19/10/18 + public void testWithVideoAndShortContentFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Arrays.asList(Filter.Video.name(), Filter.Short.name()), + "" + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element videoMatchingElement = getMatchingElement( + Filter.Video.name(), + filterList + ); + Element shortMatchingElement = getMatchingElement( + Filter.Short.name(), + filterList + ); + if(videoMatchingElement == null) { + fail("Channel filter has not been selected"); + } + if(shortMatchingElement == null) { + fail("Short filter has not been selected"); + } } @Test - public void testWithGibbershContentFilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() - .fromQuery("asdf", Collections.singletonList("gibberish"), "").getUrl()); + public void testWithChannelContentFilterAndRatingSortFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.singletonList(Filter.Channel.name()), + Sorter.Rating.name() + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element channelMatchingElement = getMatchingElement( + Filter.Channel.name(), + filterList + ); + Element ratingMatchingElement = getMatchingElement( + Sorter.Rating.name(), + filterList + ); + if(channelMatchingElement == null) { + fail("Channel filter has not been selected"); + } + if(ratingMatchingElement == null) { + fail("Rating sorter has not been selected"); + } + } + + @Test + public void testWithGibberishContentFilter() throws Exception { + assertEquals("https://www.youtube.com/results?q=" + DEFAULT_SEARCH_QUERY, YouTube.getSearchQHFactory() + .fromQuery(DEFAULT_SEARCH_QUERY, Collections.singletonList("gibberish"), "").getUrl()); } @Test public void testWithGibbershSortFilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() - .fromQuery("asdf", Collections.emptyList(), "gibberish").getUrl()); + assertEquals("https://www.youtube.com/results?q=" + DEFAULT_SEARCH_QUERY, YouTube.getSearchQHFactory() + .fromQuery(DEFAULT_SEARCH_QUERY, Collections.emptyList(), "gibberish").getUrl()); } @Test public void testGetAvailableContentFilter() { final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); - assertEquals(YoutubeSearchQueryHandlerFactory.ContentFilter.values().length, contentFilter.length); + assertEquals(Filter.values().length, contentFilter.length); } @Test public void testGetAvailableSortFilter() { final String[] sortFilters = YouTube.getSearchQHFactory().getAvailableSortFilter(); - assertEquals(YoutubeSearchQueryHandlerFactory.SortFilter.values().length, sortFilters.length); + assertEquals(Sorter.values().length, sortFilters.length); + } + + private SearchQueryHandler getYouTubeDefaultSearchQueryHandler( + List filters, + String sorter) throws Exception { + return getYouTubeSearchQueryHandler(DEFAULT_SEARCH_QUERY, filters, sorter); + } + + private SearchQueryHandler getYouTubeSearchQueryHandler( + String query, + List filters, + String sorter) throws Exception { + return YouTube.getSearchQHFactory().fromQuery(query, filters, sorter); + } + + private String getHtml(String url) throws Exception { + return NewPipe.getDownloader().download(url); + } + + private Elements getFilterList(Document document) { + return document.select("div.filter-col").select("ul"); + } + + @Nullable + private Element getMatchingElement(String matchCriteriaText, Elements elements) { + Element matchingElement = null; + for (Element element : elements) { + Elements filterElements = element.select("span.filter-text").not("span.filter-ghost"); + for (Element filterElement : filterElements) { + if (filterElement.text().contains(matchCriteriaText)) { + matchingElement = filterElement; + break; + } + } + } + + return matchingElement; } } From 93239ebfc0b44c17f7f8fa5d04013cfc62b2d880 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Fri, 9 Nov 2018 20:43:37 +0530 Subject: [PATCH 03/15] Added .java-version file to gitignore and removed it from repo --- .gitignore | 2 ++ .java-version | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 .java-version diff --git a/.gitignore b/.gitignore index bd009fd87d..cac877d61c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ gradle-app.setting # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties + +.java-version \ No newline at end of file diff --git a/.java-version b/.java-version deleted file mode 100644 index 6259340971..0000000000 --- a/.java-version +++ /dev/null @@ -1 +0,0 @@ -1.8 From 78bb9682d4f32993ba7a2c33bd2c7342dedad9c9 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sat, 10 Nov 2018 19:16:51 +0530 Subject: [PATCH 04/15] Removed usage of DataTypeConverter class and replaced it with newly added Base64 class in YoutubeSearchQueryHandlerFactory --- .../YoutubeSearchQueryHandlerFactory.java | 6 +- .../newpipe/extractor/utils/Base64.java | 725 ++++++++++++++++++ 2 files changed, 728 insertions(+), 3 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 5a15ac979e..a10d2e92a6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -2,10 +2,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Base64; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import javax.xml.bind.DatatypeConverter; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; @@ -145,7 +145,7 @@ private String getFilterQueryParams(List contentFilters, String sortFilt if(returnList.isEmpty()) { return null; } - return URLEncoder.encode(DatatypeConverter.printBase64Binary(convert(returnList)), CHARSET_UTF_8); + return URLEncoder.encode(Base64.encodeToString(convert(returnList), Base64.URL_SAFE)); } private List getContentFiltersQueryParams(List contentFilter) { @@ -188,7 +188,7 @@ private List getSortFiltersQueryParam(String filter) { sorter = Sorter.valueOf(filter); } catch (IllegalArgumentException e) { e.printStackTrace(); - System.err.println("Unknown sort filter = " + filter + ", provided, none applied."); + System.err.println("Unknown sort filter = " + filter + " provided, none applied."); return Collections.emptyList(); } return Arrays.asList(sorter.type.value, sorter.value); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java new file mode 100644 index 0000000000..c5c6eb7d67 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java @@ -0,0 +1,725 @@ +package org.schabi.newpipe.extractor.utils; + +import java.io.UnsupportedEncodingException; + +/** + * For Java 6-7, one could use the {@link javax.xml.bind.DatatypeConverter} to + * work with Base64 encoding/decoding. However the javax.xml.bind.* package is + * omitted on Android, so we cannot use this class on Android. + * + * Hence below implementation was picked from AOSP here -> + * https://github.com/aosp-mirror/platform_frameworks_base/blob/master/core/java/android/util/Base64.java + */ +public class Base64 { + /** + * Default values for encoder/decoder flags. + */ + public static final int DEFAULT = 0; + + /** + * Encoder flag bit to omit the padding '=' characters at the end + * of the output (if any). + */ + public static final int NO_PADDING = 1; + + /** + * Encoder flag bit to omit all line terminators (i.e., the output + * will be on one long line). + */ + public static final int NO_WRAP = 2; + + /** + * Encoder flag bit to indicate lines should be terminated with a + * CRLF pair instead of just an LF. Has no effect if {@code + * NO_WRAP} is specified as well. + */ + public static final int CRLF = 4; + + /** + * Encoder/decoder flag bit to indicate using the "URL and + * filename safe" variant of Base64 (see RFC 3548 section 4) where + * {@code -} and {@code _} are used in place of {@code +} and + * {@code /}. + */ + public static final int URL_SAFE = 8; + + /** + * Flag to pass to Base64OutputStream to indicate that it + * should not close the output stream it is wrapping when it + * itself is closed. + */ + public static final int NO_CLOSE = 16; + + // -------------------------------------------------------- + // shared code + // -------------------------------------------------------- + + /* package */ static abstract class Coder { + public byte[] output; + public int op; + + /** + * Encode/decode another block of input data. this.output is + * provided by the caller, and must be big enough to hold all + * the coded data. On exit, this.opwill be set to the length + * of the coded data. + * + * @param finish true if this is the final call to process for + * this object. Will finalize the coder state and + * include any final bytes in the output. + * + * @return true if the input so far is good; false if some + * error has been detected in the input stream.. + */ + public abstract boolean process(byte[] input, int offset, int len, boolean finish); + + /** + * @return the maximum number of bytes a call to process() + * could produce for the given number of input bytes. This may + * be an overestimate. + */ + public abstract int maxOutputSize(int len); + } + + // -------------------------------------------------------- + // decoding + // -------------------------------------------------------- + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + * + *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param str the input String to decode, which is converted to + * bytes using the default charset + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(String str, int flags) { + return decode(str.getBytes(), flags); + } + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + * + *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param input the input array to decode + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(byte[] input, int flags) { + return decode(input, 0, input.length, flags); + } + + /** + * Decode the Base64-encoded data in input and return the data in + * a new byte array. + * + *

The padding '=' characters at the end are considered optional, but + * if any are present, there must be the correct number of them. + * + * @param input the data to decode + * @param offset the position within the input array at which to start + * @param len the number of bytes of input to decode + * @param flags controls certain features of the decoded output. + * Pass {@code DEFAULT} to decode standard Base64. + * + * @throws IllegalArgumentException if the input contains + * incorrect padding + */ + public static byte[] decode(byte[] input, int offset, int len, int flags) { + // Allocate space for the most data the input could represent. + // (It could contain less if it contains whitespace, etc.) + Decoder decoder = new Decoder(flags, new byte[len*3/4]); + + if (!decoder.process(input, offset, len, true)) { + throw new IllegalArgumentException("bad base-64"); + } + + // Maybe we got lucky and allocated exactly enough output space. + if (decoder.op == decoder.output.length) { + return decoder.output; + } + + // Need to shorten the array, so allocate a new one of the + // right size and copy. + byte[] temp = new byte[decoder.op]; + System.arraycopy(decoder.output, 0, temp, 0, decoder.op); + return temp; + } + + /* package */ static class Decoder extends Coder { + /** + * Lookup table for turning bytes into their position in the + * Base64 alphabet. + */ + private static final int DECODE[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /** + * Decode lookup table for the "web safe" variant (RFC 3548 + * sec. 4) where - and _ replace + and /. + */ + private static final int DECODE_WEBSAFE[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /** Non-data values in the DECODE arrays. */ + private static final int SKIP = -1; + private static final int EQUALS = -2; + + /** + * States 0-3 are reading through the next input tuple. + * State 4 is having read one '=' and expecting exactly + * one more. + * State 5 is expecting no more data or padding characters + * in the input. + * State 6 is the error state; an error has been detected + * in the input and no future input can "fix" it. + */ + private int state; // state number (0 to 6) + private int value; + + final private int[] alphabet; + + public Decoder(int flags, byte[] output) { + this.output = output; + + alphabet = ((flags & URL_SAFE) == 0) ? DECODE : DECODE_WEBSAFE; + state = 0; + value = 0; + } + + /** + * @return an overestimate for the number of bytes {@code + * len} bytes could decode to. + */ + public int maxOutputSize(int len) { + return len * 3/4 + 10; + } + + /** + * Decode another block of input data. + * + * @return true if the state machine is still healthy. false if + * bad base-64 data has been detected in the input stream. + */ + public boolean process(byte[] input, int offset, int len, boolean finish) { + if (this.state == 6) return false; + + int p = offset; + len += offset; + + // Using local variables makes the decoder about 12% + // faster than if we manipulate the member variables in + // the loop. (Even alphabet makes a measurable + // difference, which is somewhat surprising to me since + // the member variable is final.) + int state = this.state; + int value = this.value; + int op = 0; + final byte[] output = this.output; + final int[] alphabet = this.alphabet; + + while (p < len) { + // Try the fast path: we're starting a new tuple and the + // next four bytes of the input stream are all data + // bytes. This corresponds to going through states + // 0-1-2-3-0. We expect to use this method for most of + // the data. + // + // If any of the next four bytes of input are non-data + // (whitespace, etc.), value will end up negative. (All + // the non-data values in decode are small negative + // numbers, so shifting any of them up and or'ing them + // together will result in a value with its top bit set.) + // + // You can remove this whole block and the output should + // be the same, just slower. + if (state == 0) { + while (p+4 <= len && + (value = ((alphabet[input[p] & 0xff] << 18) | + (alphabet[input[p+1] & 0xff] << 12) | + (alphabet[input[p+2] & 0xff] << 6) | + (alphabet[input[p+3] & 0xff]))) >= 0) { + output[op+2] = (byte) value; + output[op+1] = (byte) (value >> 8); + output[op] = (byte) (value >> 16); + op += 3; + p += 4; + } + if (p >= len) break; + } + + // The fast path isn't available -- either we've read a + // partial tuple, or the next four input bytes aren't all + // data, or whatever. Fall back to the slower state + // machine implementation. + + int d = alphabet[input[p++] & 0xff]; + + switch (state) { + case 0: + if (d >= 0) { + value = d; + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 1: + if (d >= 0) { + value = (value << 6) | d; + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 2: + if (d >= 0) { + value = (value << 6) | d; + ++state; + } else if (d == EQUALS) { + // Emit the last (partial) output tuple; + // expect exactly one more padding character. + output[op++] = (byte) (value >> 4); + state = 4; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 3: + if (d >= 0) { + // Emit the output triple and return to state 0. + value = (value << 6) | d; + output[op+2] = (byte) value; + output[op+1] = (byte) (value >> 8); + output[op] = (byte) (value >> 16); + op += 3; + state = 0; + } else if (d == EQUALS) { + // Emit the last (partial) output tuple; + // expect no further data or padding characters. + output[op+1] = (byte) (value >> 2); + output[op] = (byte) (value >> 10); + op += 2; + state = 5; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 4: + if (d == EQUALS) { + ++state; + } else if (d != SKIP) { + this.state = 6; + return false; + } + break; + + case 5: + if (d != SKIP) { + this.state = 6; + return false; + } + break; + } + } + + if (!finish) { + // We're out of input, but a future call could provide + // more. + this.state = state; + this.value = value; + this.op = op; + return true; + } + + // Done reading input. Now figure out where we are left in + // the state machine and finish up. + + switch (state) { + case 0: + // Output length is a multiple of three. Fine. + break; + case 1: + // Read one extra input byte, which isn't enough to + // make another output byte. Illegal. + this.state = 6; + return false; + case 2: + // Read two extra input bytes, enough to emit 1 more + // output byte. Fine. + output[op++] = (byte) (value >> 4); + break; + case 3: + // Read three extra input bytes, enough to emit 2 more + // output bytes. Fine. + output[op++] = (byte) (value >> 10); + output[op++] = (byte) (value >> 2); + break; + case 4: + // Read one padding '=' when we expected 2. Illegal. + this.state = 6; + return false; + case 5: + // Read all the padding '='s we expected and no more. + // Fine. + break; + } + + this.state = state; + this.op = op; + return true; + } + } + + // -------------------------------------------------------- + // encoding + // -------------------------------------------------------- + + /** + * Base64-encode the given data and return a newly allocated + * String with the result. + * + * @param input the data to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static String encodeToString(byte[] input, int flags) { + try { + return new String(encode(input, flags), "US-ASCII"); + } catch (UnsupportedEncodingException e) { + // US-ASCII is guaranteed to be available. + throw new AssertionError(e); + } + } + + /** + * Base64-encode the given data and return a newly allocated + * String with the result. + * + * @param input the data to encode + * @param offset the position within the input array at which to + * start + * @param len the number of bytes of input to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static String encodeToString(byte[] input, int offset, int len, int flags) { + try { + return new String(encode(input, offset, len, flags), "US-ASCII"); + } catch (UnsupportedEncodingException e) { + // US-ASCII is guaranteed to be available. + throw new AssertionError(e); + } + } + + /** + * Base64-encode the given data and return a newly allocated + * byte[] with the result. + * + * @param input the data to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static byte[] encode(byte[] input, int flags) { + return encode(input, 0, input.length, flags); + } + + /** + * Base64-encode the given data and return a newly allocated + * byte[] with the result. + * + * @param input the data to encode + * @param offset the position within the input array at which to + * start + * @param len the number of bytes of input to encode + * @param flags controls certain features of the encoded output. + * Passing {@code DEFAULT} results in output that + * adheres to RFC 2045. + */ + public static byte[] encode(byte[] input, int offset, int len, int flags) { + Encoder encoder = new Encoder(flags, null); + + // Compute the exact length of the array we will produce. + int output_len = len / 3 * 4; + + // Account for the tail of the data and the padding bytes, if any. + if (encoder.do_padding) { + if (len % 3 > 0) { + output_len += 4; + } + } else { + switch (len % 3) { + case 0: break; + case 1: output_len += 2; break; + case 2: output_len += 3; break; + } + } + + // Account for the newlines, if any. + if (encoder.do_newline && len > 0) { + output_len += (((len-1) / (3 * Encoder.LINE_GROUPS)) + 1) * + (encoder.do_cr ? 2 : 1); + } + + encoder.output = new byte[output_len]; + encoder.process(input, offset, len, true); + + assert encoder.op == output_len; + + return encoder.output; + } + + /* package */ static class Encoder extends Coder { + /** + * Emit a new line every this many output tuples. Corresponds to + * a 76-character line length (the maximum allowable according to + * RFC 2045). + */ + public static final int LINE_GROUPS = 19; + + /** + * Lookup table for turning Base64 alphabet positions (6 bits) + * into output bytes. + */ + private static final byte ENCODE[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', + }; + + /** + * Lookup table for turning Base64 alphabet positions (6 bits) + * into output bytes. + */ + private static final byte ENCODE_WEBSAFE[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', + }; + + final private byte[] tail; + /* package */ int tailLen; + private int count; + + final public boolean do_padding; + final public boolean do_newline; + final public boolean do_cr; + final private byte[] alphabet; + + public Encoder(int flags, byte[] output) { + this.output = output; + + do_padding = (flags & NO_PADDING) == 0; + do_newline = (flags & NO_WRAP) == 0; + do_cr = (flags & CRLF) != 0; + alphabet = ((flags & URL_SAFE) == 0) ? ENCODE : ENCODE_WEBSAFE; + + tail = new byte[2]; + tailLen = 0; + + count = do_newline ? LINE_GROUPS : -1; + } + + /** + * @return an overestimate for the number of bytes {@code + * len} bytes could encode to. + */ + public int maxOutputSize(int len) { + return len * 8/5 + 10; + } + + public boolean process(byte[] input, int offset, int len, boolean finish) { + // Using local variables makes the encoder about 9% faster. + final byte[] alphabet = this.alphabet; + final byte[] output = this.output; + int op = 0; + int count = this.count; + + int p = offset; + len += offset; + int v = -1; + + // First we need to concatenate the tail of the previous call + // with any input bytes available now and see if we can empty + // the tail. + + switch (tailLen) { + case 0: + // There was no tail. + break; + + case 1: + if (p+2 <= len) { + // A 1-byte tail with at least 2 bytes of + // input available now. + v = ((tail[0] & 0xff) << 16) | + ((input[p++] & 0xff) << 8) | + (input[p++] & 0xff); + tailLen = 0; + }; + break; + + case 2: + if (p+1 <= len) { + // A 2-byte tail with at least 1 byte of input. + v = ((tail[0] & 0xff) << 16) | + ((tail[1] & 0xff) << 8) | + (input[p++] & 0xff); + tailLen = 0; + } + break; + } + + if (v != -1) { + output[op++] = alphabet[(v >> 18) & 0x3f]; + output[op++] = alphabet[(v >> 12) & 0x3f]; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (--count == 0) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + count = LINE_GROUPS; + } + } + + // At this point either there is no tail, or there are fewer + // than 3 bytes of input available. + + // The main loop, turning 3 input bytes into 4 output bytes on + // each iteration. + while (p+3 <= len) { + v = ((input[p] & 0xff) << 16) | + ((input[p+1] & 0xff) << 8) | + (input[p+2] & 0xff); + output[op] = alphabet[(v >> 18) & 0x3f]; + output[op+1] = alphabet[(v >> 12) & 0x3f]; + output[op+2] = alphabet[(v >> 6) & 0x3f]; + output[op+3] = alphabet[v & 0x3f]; + p += 3; + op += 4; + if (--count == 0) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + count = LINE_GROUPS; + } + } + + if (finish) { + // Finish up the tail of the input. Note that we need to + // consume any bytes in tail before any bytes + // remaining in input; there should be at most two bytes + // total. + + if (p-tailLen == len-1) { + int t = 0; + v = ((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 4; + tailLen -= t; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (do_padding) { + output[op++] = '='; + output[op++] = '='; + } + if (do_newline) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + } else if (p-tailLen == len-2) { + int t = 0; + v = (((tailLen > 1 ? tail[t++] : input[p++]) & 0xff) << 10) | + (((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 2); + tailLen -= t; + output[op++] = alphabet[(v >> 12) & 0x3f]; + output[op++] = alphabet[(v >> 6) & 0x3f]; + output[op++] = alphabet[v & 0x3f]; + if (do_padding) { + output[op++] = '='; + } + if (do_newline) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + } else if (do_newline && op > 0 && count != LINE_GROUPS) { + if (do_cr) output[op++] = '\r'; + output[op++] = '\n'; + } + + assert tailLen == 0; + assert p == len; + } else { + // Save the leftovers in tail to be consumed on the next + // call to encodeInternal. + + if (p == len-1) { + tail[tailLen++] = input[p]; + } else if (p == len-2) { + tail[tailLen++] = input[p]; + tail[tailLen++] = input[p+1]; + } + } + + this.op = op; + this.count = count; + + return true; + } + } +} From f71b62f99f742f58ae408b03bf3789d66d43752d Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sat, 8 Dec 2018 22:03:44 +0530 Subject: [PATCH 05/15] Changes made: - Fixed indendation issue in YoutubeSearchQueryHandlerFactory and YoutubeSearchQHTest. - Added Features filter type and filter. --- .../YoutubeSearchQueryHandlerFactory.java | 112 ++++++++++-------- .../youtube/search/YoutubeSearchQHTest.java | 12 +- 2 files changed, 70 insertions(+), 54 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index a10d2e92a6..9ed222026e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -18,9 +18,10 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public static final String CHARSET_UTF_8 = "UTF-8"; public enum FilterType { - Content((byte)0x10), - Time((byte)0x08), - Duration((byte)0x18); + Content((byte) 0x10), + Time((byte) 0x08), + Duration((byte) 0x18), + Feature((byte) 0); private final byte value; @@ -30,33 +31,46 @@ public enum FilterType { } public enum Filter { - All(FilterType.Content,(byte)0), - Video(FilterType.Content,(byte)0x01), - Channel(FilterType.Content,(byte)0x02), - Playlist(FilterType.Content,(byte)0x03), - Movie(FilterType.Content,(byte)0x04), - Show(FilterType.Content,(byte)0x05), - - Hour(FilterType.Time,(byte)0x01), - Today(FilterType.Time,(byte)0x02), - Week(FilterType.Time,(byte)0x03), - Month(FilterType.Time,(byte)0x04), - Year(FilterType.Time,(byte)0x05), - - Short(FilterType.Duration, (byte)0x01), - Long(FilterType.Duration, (byte)0x02); - + All("All", FilterType.Content, (byte) 0), + Video("Video", FilterType.Content, (byte) 0x01), + Channel("Channel", FilterType.Content, (byte) 0x02), + Playlist("Playlist", FilterType.Content, (byte) 0x03), + Movie("Movie", FilterType.Content, (byte) 0x04), + Show("Show", FilterType.Content, (byte) 0x05), + + Hour("Hour", FilterType.Time, (byte) 0x01), + Today("Today", FilterType.Time, (byte) 0x02), + Week("Week", FilterType.Time, (byte) 0x03), + Month("Month", FilterType.Time, (byte) 0x04), + Year("Year", FilterType.Time, (byte) 0x05), + + Short("Short", FilterType.Duration, (byte) 0x01), + Long("Long", FilterType.Duration, (byte) 0x02), + + HD("HD", FilterType.Feature, (byte) 0x2001), + Subtitles("Subtitles", FilterType.Feature, (byte) 0x2801), + CreativeCommons("Creative Commons", FilterType.Feature, (byte) 0x3001), + ThreeDimensional("3D", FilterType.Feature, (byte) 0x3801), + Live("Live", FilterType.Feature, (byte) 0x4001), + Purchased("Purchased", FilterType.Feature, (byte) 0x4801), + FourK("4k", FilterType.Feature, (byte) 0x7001), + ThreeSixty("360", FilterType.Feature, (byte) 0x7801), + Location("Location", FilterType.Feature, (byte) 0xb80101), + HDR("HDR", FilterType.Feature, (byte) 0xc80101); + + private final String title; private final FilterType type; private final byte value; - Filter(FilterType type, byte value) { + Filter(String title, FilterType type, byte value) { + this.title = title; this.type = type; this.value = value; } } public enum SorterType { - Default((byte)0x08); + Default((byte) 0x08); private final byte value; @@ -66,15 +80,17 @@ public enum SorterType { } public enum Sorter { - Relevance(SorterType.Default, (byte)0x00), - Rating(SorterType.Default, (byte)0x01), - Upload_Date(SorterType.Default, (byte)0x02), - View_Count(SorterType.Default, (byte)0x03); + Relevance("Relevance", SorterType.Default, (byte) 0x00), + Rating("Rating", SorterType.Default, (byte) 0x01), + Upload_Date("Upload_Date", SorterType.Default, (byte) 0x02), + View_Count("View_Count", SorterType.Default, (byte) 0x03); + private final String title; private final SorterType type; private final byte value; - Sorter(SorterType type, byte value) { + Sorter(String title, SorterType type, byte value) { + this.title = title; this.type = type; this.value = value; } @@ -89,20 +105,22 @@ public String getUrl(String searchString, List contentFilters, String so try { String returnURL = getSearchBaseUrl(searchString); String filterQueryParams = getFilterQueryParams(contentFilters, sortFilter); - if(filterQueryParams != null) { + if (filterQueryParams != null) { returnURL = returnURL + "&sp=" + filterQueryParams; } return returnURL; } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); + } catch (IllegalArgumentException e) { + throw new ParsingException("Failed to get search results", e); } } @Override public String[] getAvailableContentFilter() { List contentFiltersList = new ArrayList<>(); - for(Filter contentFilter : Filter.values()) { - contentFiltersList.add(contentFilter.name()); + for (Filter contentFilter : Filter.values()) { + contentFiltersList.add(contentFilter.title); } String[] contentFiltersArray = new String[contentFiltersList.size()]; contentFiltersArray = contentFiltersList.toArray(contentFiltersArray); @@ -112,8 +130,8 @@ public String[] getAvailableContentFilter() { @Override public String[] getAvailableSortFilter() { List sortFiltersList = new ArrayList<>(); - for(Sorter sortFilter : Sorter.values()) { - sortFiltersList.add(sortFilter.name()); + for (Sorter sortFilter : Sorter.values()) { + sortFiltersList.add(sortFilter.title); } String[] sortFiltersArray = new String[sortFiltersList.size()]; sortFiltersArray = sortFiltersList.toArray(sortFiltersArray); @@ -129,47 +147,46 @@ private String getSearchBaseUrl(String searchQuery) @Nullable private String getFilterQueryParams(List contentFilters, String sortFilter) - throws UnsupportedEncodingException { + throws IllegalArgumentException { List returnList = new ArrayList<>(); List sortFilterParams = getSortFiltersQueryParam(sortFilter); - if(!sortFilterParams.isEmpty()) { + if (!sortFilterParams.isEmpty()) { returnList.addAll(sortFilterParams); } List contentFilterParams = getContentFiltersQueryParams(contentFilters); - if(!contentFilterParams.isEmpty()) { - returnList.add((byte)0x12); - returnList.add((byte)contentFilterParams.size()); + if (!contentFilterParams.isEmpty()) { + returnList.add((byte) 0x12); + returnList.add((byte) contentFilterParams.size()); returnList.addAll(contentFilterParams); } - if(returnList.isEmpty()) { + if (returnList.isEmpty()) { return null; } return URLEncoder.encode(Base64.encodeToString(convert(returnList), Base64.URL_SAFE)); } - private List getContentFiltersQueryParams(List contentFilter) { - if(contentFilter == null || contentFilter.isEmpty()) { + private List getContentFiltersQueryParams(List contentFilter) throws IllegalArgumentException { + if (contentFilter == null || contentFilter.isEmpty()) { return Collections.emptyList(); } List returnList = new ArrayList<>(); - for(String filter : contentFilter) { + for (String filter : contentFilter) { List byteList = getContentFilterQueryParams(filter); - if(!byteList.isEmpty()) { + if (!byteList.isEmpty()) { returnList.addAll(byteList); } } return returnList; } - private List getContentFilterQueryParams(String filter) { + private List getContentFilterQueryParams(String filter) throws IllegalArgumentException { Filter contentFilter; try { contentFilter = Filter.valueOf(filter); } catch (IllegalArgumentException iae) { iae.printStackTrace(); - System.err.println("Unknown content filter type provided = " + filter +", none will be applied"); - return Collections.emptyList(); + throw new IllegalArgumentException("Unknown content filter type provided = " + filter + ", none will be applied"); } switch (contentFilter) { case All: @@ -179,8 +196,8 @@ private List getContentFilterQueryParams(String filter) { } } - private List getSortFiltersQueryParam(String filter) { - if(filter == null || filter.isEmpty()) { + private List getSortFiltersQueryParam(String filter) throws IllegalArgumentException { + if (filter == null || filter.isEmpty()) { return Collections.emptyList(); } Sorter sorter; @@ -188,8 +205,7 @@ private List getSortFiltersQueryParam(String filter) { sorter = Sorter.valueOf(filter); } catch (IllegalArgumentException e) { e.printStackTrace(); - System.err.println("Unknown sort filter = " + filter + " provided, none applied."); - return Collections.emptyList(); + throw new IllegalArgumentException("Unknown sort filter = " + filter + " provided, none applied."); } return Arrays.asList(sorter.type.value, sorter.value); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 20f18e9896..81617c3818 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -62,7 +62,7 @@ public void testWithChannelContentFilter() throws Exception { Filter.Channel.name(), filterList ); - if(matchingElement == null) { + if (matchingElement == null) { fail("Channel filter has not been selected"); } } @@ -80,7 +80,7 @@ public void testWithRatingSortFilter() throws Exception { Sorter.Rating.name(), filterList ); - if(matchingElement == null) { + if (matchingElement == null) { fail("Rating sorter has not been selected"); } } @@ -102,10 +102,10 @@ public void testWithVideoAndShortContentFilter() throws Exception { Filter.Short.name(), filterList ); - if(videoMatchingElement == null) { + if (videoMatchingElement == null) { fail("Channel filter has not been selected"); } - if(shortMatchingElement == null) { + if (shortMatchingElement == null) { fail("Short filter has not been selected"); } } @@ -127,10 +127,10 @@ public void testWithChannelContentFilterAndRatingSortFilter() throws Exception { Sorter.Rating.name(), filterList ); - if(channelMatchingElement == null) { + if (channelMatchingElement == null) { fail("Channel filter has not been selected"); } - if(ratingMatchingElement == null) { + if (ratingMatchingElement == null) { fail("Rating sorter has not been selected"); } } From 75ad8f9f80eebde0e3921911635c023f3f067357 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sun, 9 Dec 2018 00:12:17 +0530 Subject: [PATCH 06/15] Changes made: - Fixed handling for features content filters. - Added test cases to check for whether feature content filters are working fine. --- .../YoutubeSearchQueryHandlerFactory.java | 89 +++++++++++-------- .../youtube/search/YoutubeSearchQHTest.java | 61 +++++++++++-- 2 files changed, 105 insertions(+), 45 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 9ed222026e..e97b5fb30f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -18,54 +18,58 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory public static final String CHARSET_UTF_8 = "UTF-8"; public enum FilterType { - Content((byte) 0x10), - Time((byte) 0x08), - Duration((byte) 0x18), - Feature((byte) 0); + Content(new byte[]{0x10}), + Time(new byte[]{0x08}), + Duration(new byte[]{0x18}), + Feature(new byte[]{}); - private final byte value; + private final byte[] values; - FilterType(byte value) { - this.value = value; + FilterType(byte[] values) { + this.values = values; } } public enum Filter { - All("All", FilterType.Content, (byte) 0), - Video("Video", FilterType.Content, (byte) 0x01), - Channel("Channel", FilterType.Content, (byte) 0x02), - Playlist("Playlist", FilterType.Content, (byte) 0x03), - Movie("Movie", FilterType.Content, (byte) 0x04), - Show("Show", FilterType.Content, (byte) 0x05), - - Hour("Hour", FilterType.Time, (byte) 0x01), - Today("Today", FilterType.Time, (byte) 0x02), - Week("Week", FilterType.Time, (byte) 0x03), - Month("Month", FilterType.Time, (byte) 0x04), - Year("Year", FilterType.Time, (byte) 0x05), - - Short("Short", FilterType.Duration, (byte) 0x01), - Long("Long", FilterType.Duration, (byte) 0x02), - - HD("HD", FilterType.Feature, (byte) 0x2001), - Subtitles("Subtitles", FilterType.Feature, (byte) 0x2801), - CreativeCommons("Creative Commons", FilterType.Feature, (byte) 0x3001), - ThreeDimensional("3D", FilterType.Feature, (byte) 0x3801), - Live("Live", FilterType.Feature, (byte) 0x4001), - Purchased("Purchased", FilterType.Feature, (byte) 0x4801), - FourK("4k", FilterType.Feature, (byte) 0x7001), - ThreeSixty("360", FilterType.Feature, (byte) 0x7801), - Location("Location", FilterType.Feature, (byte) 0xb80101), - HDR("HDR", FilterType.Feature, (byte) 0xc80101); + All("All", FilterType.Content, new byte[]{0}), + Video("Video", FilterType.Content, new byte[]{0x01}), + Channel("Channel", FilterType.Content, new byte[]{0x02}), + Playlist("Playlist", FilterType.Content, new byte[]{0x03}), + Movie("Movie", FilterType.Content, new byte[]{0x04}), + Show("Show", FilterType.Content, new byte[]{0x05}), + + Hour("Hour", FilterType.Time, new byte[]{0x01}), + Today("Today", FilterType.Time, new byte[]{0x02}), + Week("Week", FilterType.Time, new byte[]{0x03}), + Month("Month", FilterType.Time, new byte[]{0x04}), + Year("Year", FilterType.Time, new byte[]{0x05}), + + Short("Short", FilterType.Duration, new byte[]{0x01}), + Long("Long", FilterType.Duration, new byte[]{0x02}), + + HD("HD", FilterType.Feature, new byte[]{0x20, 0x01}), + Subtitles("Subtitles", FilterType.Feature, new byte[]{0x28, 0x01}), + CreativeCommons("Creative Commons", FilterType.Feature, new byte[]{0x30, 0x01}), + ThreeDimensional("3D", FilterType.Feature, new byte[]{0x38, 0x01}), + Live("Live", FilterType.Feature, new byte[]{0x40, 0x01}), + Purchased("Purchased", FilterType.Feature, new byte[]{0x48, 0x01}), + FourK("4k", FilterType.Feature, new byte[]{0x70, 0x01}), + ThreeSixty("360", FilterType.Feature, new byte[]{0x78, 0x01}), + Location("Location", FilterType.Feature, new byte[]{(byte) 0xb8, 0x01, 0x01}), + HDR("HDR", FilterType.Feature, new byte[]{(byte) 0xc8, 0x01, 0x01}); private final String title; private final FilterType type; - private final byte value; + private final byte[] values; - Filter(String title, FilterType type, byte value) { + Filter(String title, FilterType type, byte[] values) { this.title = title; this.type = type; - this.value = value; + this.values = values; + } + + public String getTitle() { + return title; } } @@ -192,7 +196,10 @@ private List getContentFilterQueryParams(String filter) throws IllegalArgu case All: return Collections.emptyList(); default: - return Arrays.asList(contentFilter.type.value, contentFilter.value); + List returnList = new ArrayList<>(); + returnList.addAll(convert(contentFilter.type.values)); + returnList.addAll(convert(contentFilter.values)); + return returnList; } } @@ -217,4 +224,12 @@ private byte[] convert(@Nonnull List bigByteList) { } return returnArray; } + + private List convert(@Nonnull byte[] byteArray) { + List returnList = new ArrayList<>(byteArray.length); + for (int i = 0; i < byteArray.length; i++) { + returnList.add(i, byteArray[0]); + } + return returnList; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 81617c3818..6c80d3e2db 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -8,13 +8,13 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Filter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.Sorter; import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -67,6 +67,49 @@ public void testWithChannelContentFilter() throws Exception { } } + @Test + public void testWith360ContentFilterAndNoSortFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.singletonList(Filter.ThreeSixty.name()), + "" + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element matchingElement = getMatchingElement( + Filter.ThreeSixty.getTitle(), + filterList + ); + if (matchingElement == null) { + fail("360 filter has not been selected"); + } + } + + @Test + public void testWith360ContentFilterAndRatingSortFilter() throws Exception { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.singletonList(Filter.ThreeSixty.name()), + Sorter.Rating.name() + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element threeSixtyMatchingElement = getMatchingElement( + Filter.ThreeSixty.getTitle(), + filterList + ); + Element ratingMatchingElement = getMatchingElement( + Sorter.Rating.name(), + filterList + ); + if (threeSixtyMatchingElement == null) { + fail("360 filter has not been selected"); + } + if (ratingMatchingElement == null) { + fail("Rating sorter has not been selected"); + } + } + @Test public void testWithRatingSortFilter() throws Exception { String url = getYouTubeDefaultSearchQueryHandler( @@ -135,16 +178,18 @@ public void testWithChannelContentFilterAndRatingSortFilter() throws Exception { } } - @Test + @Test(expected = ParsingException.class) public void testWithGibberishContentFilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=" + DEFAULT_SEARCH_QUERY, YouTube.getSearchQHFactory() - .fromQuery(DEFAULT_SEARCH_QUERY, Collections.singletonList("gibberish"), "").getUrl()); + YouTube.getSearchQHFactory() + .fromQuery(DEFAULT_SEARCH_QUERY, Collections.singletonList("gibberish"), "") + .getUrl(); } - @Test - public void testWithGibbershSortFilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=" + DEFAULT_SEARCH_QUERY, YouTube.getSearchQHFactory() - .fromQuery(DEFAULT_SEARCH_QUERY, Collections.emptyList(), "gibberish").getUrl()); + @Test(expected = ParsingException.class) + public void testWithGibberishSortFilter() throws Exception { + YouTube.getSearchQHFactory() + .fromQuery(DEFAULT_SEARCH_QUERY, Collections.emptyList(), "gibberish") + .getUrl(); } @Test From 98b67a49e6737af97bf115c6355e115cf705b2ed Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sun, 16 Dec 2018 10:54:00 +0530 Subject: [PATCH 07/15] Changes made: - Added license header to Base64 class. - Added Apache license for AOSP project in a different directory. --- .../newpipe/extractor/utils/Base64.java | 15 ++ third-party-licenses/AOSP-APACHE-LICENSE-V2.0 | 228 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 third-party-licenses/AOSP-APACHE-LICENSE-V2.0 diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java index c5c6eb7d67..45e8d6cde6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.schabi.newpipe.extractor.utils; import java.io.UnsupportedEncodingException; diff --git a/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 b/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 new file mode 100644 index 0000000000..c8a8c53b2c --- /dev/null +++ b/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 @@ -0,0 +1,228 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + + +UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE + +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +and http://www.unicode.org/cldr/data/ . Unicode Software includes any +source code published in the Unicode Standard or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, and +http://www.unicode.org/cldr/data/. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA +FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY +ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF +THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, +DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2008 Unicode, Inc. All rights reserved. Distributed +under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation (the +"Data Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, +and to permit persons to whom the Data Files or Software are furnished to +do so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File +or in the Software as well as in the documentation associated with the +Data File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT +OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE +OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in these Data Files or Software without prior written +authorization of the copyright holder. From b02ca4ace3a5633bd0a27092e5a37b60b456092d Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sun, 16 Dec 2018 22:36:40 +0530 Subject: [PATCH 08/15] Added top level comment explaining modifications made to the Base64 class file --- .../main/java/org/schabi/newpipe/extractor/utils/Base64.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java index 45e8d6cde6..b34409536b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java @@ -1,4 +1,8 @@ /* + * Added a comment explaining why the file/class was pulled + * into the NewPipe project in the first place along with a URL + * pointing to the source. + * * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); From 6e8515d3fdff16d7cd13b531841fa2d058209aaf Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Thu, 3 Jan 2019 23:07:11 +0530 Subject: [PATCH 09/15] Fix bug where index is used incorrectly --- .../youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index e97b5fb30f..1ab58a2c91 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -228,7 +228,7 @@ private byte[] convert(@Nonnull List bigByteList) { private List convert(@Nonnull byte[] byteArray) { List returnList = new ArrayList<>(byteArray.length); for (int i = 0; i < byteArray.length; i++) { - returnList.add(i, byteArray[0]); + returnList.add(i, byteArray[i]); } return returnList; } From cd38c1bec709ff06556d1835b9452dc43ab25f32 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sat, 23 Mar 2019 20:18:26 +0530 Subject: [PATCH 10/15] Replace Base64 class introduced with Base64 encoding implementation off Apache Commons Codec library --- extractor/build.gradle | 2 +- .../YoutubeSearchQueryHandlerFactory.java | 9 +- .../newpipe/extractor/utils/Base64.java | 744 ------------------ third-party-licenses/AOSP-APACHE-LICENSE-V2.0 | 228 ------ 4 files changed, 6 insertions(+), 977 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java delete mode 100644 third-party-licenses/AOSP-APACHE-LICENSE-V2.0 diff --git a/extractor/build.gradle b/extractor/build.gradle index 1b7fbf001f..4962f08013 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -6,6 +6,6 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' - + implementation 'commons-codec:commons-codec:1.9' testImplementation 'junit:junit:4.12' } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 1ab58a2c91..670666533f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -1,11 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import org.apache.commons.codec.binary.Base64; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; -import org.schabi.newpipe.extractor.utils.Base64; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; @@ -115,7 +116,7 @@ public String getUrl(String searchString, List contentFilters, String so return returnURL; } catch (UnsupportedEncodingException e) { throw new ParsingException("Could not encode query", e); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | IOException e) { throw new ParsingException("Failed to get search results", e); } } @@ -151,7 +152,7 @@ private String getSearchBaseUrl(String searchQuery) @Nullable private String getFilterQueryParams(List contentFilters, String sortFilter) - throws IllegalArgumentException { + throws IllegalArgumentException, IOException { List returnList = new ArrayList<>(); List sortFilterParams = getSortFiltersQueryParam(sortFilter); if (!sortFilterParams.isEmpty()) { @@ -167,7 +168,7 @@ private String getFilterQueryParams(List contentFilters, String sortFilt if (returnList.isEmpty()) { return null; } - return URLEncoder.encode(Base64.encodeToString(convert(returnList), Base64.URL_SAFE)); + return URLEncoder.encode(Base64.encodeBase64String(convert(returnList)), "UTF-8"); } private List getContentFiltersQueryParams(List contentFilter) throws IllegalArgumentException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java deleted file mode 100644 index b34409536b..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64.java +++ /dev/null @@ -1,744 +0,0 @@ -/* - * Added a comment explaining why the file/class was pulled - * into the NewPipe project in the first place along with a URL - * pointing to the source. - * - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.schabi.newpipe.extractor.utils; - -import java.io.UnsupportedEncodingException; - -/** - * For Java 6-7, one could use the {@link javax.xml.bind.DatatypeConverter} to - * work with Base64 encoding/decoding. However the javax.xml.bind.* package is - * omitted on Android, so we cannot use this class on Android. - * - * Hence below implementation was picked from AOSP here -> - * https://github.com/aosp-mirror/platform_frameworks_base/blob/master/core/java/android/util/Base64.java - */ -public class Base64 { - /** - * Default values for encoder/decoder flags. - */ - public static final int DEFAULT = 0; - - /** - * Encoder flag bit to omit the padding '=' characters at the end - * of the output (if any). - */ - public static final int NO_PADDING = 1; - - /** - * Encoder flag bit to omit all line terminators (i.e., the output - * will be on one long line). - */ - public static final int NO_WRAP = 2; - - /** - * Encoder flag bit to indicate lines should be terminated with a - * CRLF pair instead of just an LF. Has no effect if {@code - * NO_WRAP} is specified as well. - */ - public static final int CRLF = 4; - - /** - * Encoder/decoder flag bit to indicate using the "URL and - * filename safe" variant of Base64 (see RFC 3548 section 4) where - * {@code -} and {@code _} are used in place of {@code +} and - * {@code /}. - */ - public static final int URL_SAFE = 8; - - /** - * Flag to pass to Base64OutputStream to indicate that it - * should not close the output stream it is wrapping when it - * itself is closed. - */ - public static final int NO_CLOSE = 16; - - // -------------------------------------------------------- - // shared code - // -------------------------------------------------------- - - /* package */ static abstract class Coder { - public byte[] output; - public int op; - - /** - * Encode/decode another block of input data. this.output is - * provided by the caller, and must be big enough to hold all - * the coded data. On exit, this.opwill be set to the length - * of the coded data. - * - * @param finish true if this is the final call to process for - * this object. Will finalize the coder state and - * include any final bytes in the output. - * - * @return true if the input so far is good; false if some - * error has been detected in the input stream.. - */ - public abstract boolean process(byte[] input, int offset, int len, boolean finish); - - /** - * @return the maximum number of bytes a call to process() - * could produce for the given number of input bytes. This may - * be an overestimate. - */ - public abstract int maxOutputSize(int len); - } - - // -------------------------------------------------------- - // decoding - // -------------------------------------------------------- - - /** - * Decode the Base64-encoded data in input and return the data in - * a new byte array. - * - *

The padding '=' characters at the end are considered optional, but - * if any are present, there must be the correct number of them. - * - * @param str the input String to decode, which is converted to - * bytes using the default charset - * @param flags controls certain features of the decoded output. - * Pass {@code DEFAULT} to decode standard Base64. - * - * @throws IllegalArgumentException if the input contains - * incorrect padding - */ - public static byte[] decode(String str, int flags) { - return decode(str.getBytes(), flags); - } - - /** - * Decode the Base64-encoded data in input and return the data in - * a new byte array. - * - *

The padding '=' characters at the end are considered optional, but - * if any are present, there must be the correct number of them. - * - * @param input the input array to decode - * @param flags controls certain features of the decoded output. - * Pass {@code DEFAULT} to decode standard Base64. - * - * @throws IllegalArgumentException if the input contains - * incorrect padding - */ - public static byte[] decode(byte[] input, int flags) { - return decode(input, 0, input.length, flags); - } - - /** - * Decode the Base64-encoded data in input and return the data in - * a new byte array. - * - *

The padding '=' characters at the end are considered optional, but - * if any are present, there must be the correct number of them. - * - * @param input the data to decode - * @param offset the position within the input array at which to start - * @param len the number of bytes of input to decode - * @param flags controls certain features of the decoded output. - * Pass {@code DEFAULT} to decode standard Base64. - * - * @throws IllegalArgumentException if the input contains - * incorrect padding - */ - public static byte[] decode(byte[] input, int offset, int len, int flags) { - // Allocate space for the most data the input could represent. - // (It could contain less if it contains whitespace, etc.) - Decoder decoder = new Decoder(flags, new byte[len*3/4]); - - if (!decoder.process(input, offset, len, true)) { - throw new IllegalArgumentException("bad base-64"); - } - - // Maybe we got lucky and allocated exactly enough output space. - if (decoder.op == decoder.output.length) { - return decoder.output; - } - - // Need to shorten the array, so allocate a new one of the - // right size and copy. - byte[] temp = new byte[decoder.op]; - System.arraycopy(decoder.output, 0, temp, 0, decoder.op); - return temp; - } - - /* package */ static class Decoder extends Coder { - /** - * Lookup table for turning bytes into their position in the - * Base64 alphabet. - */ - private static final int DECODE[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - }; - - /** - * Decode lookup table for the "web safe" variant (RFC 3548 - * sec. 4) where - and _ replace + and /. - */ - private static final int DECODE_WEBSAFE[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - }; - - /** Non-data values in the DECODE arrays. */ - private static final int SKIP = -1; - private static final int EQUALS = -2; - - /** - * States 0-3 are reading through the next input tuple. - * State 4 is having read one '=' and expecting exactly - * one more. - * State 5 is expecting no more data or padding characters - * in the input. - * State 6 is the error state; an error has been detected - * in the input and no future input can "fix" it. - */ - private int state; // state number (0 to 6) - private int value; - - final private int[] alphabet; - - public Decoder(int flags, byte[] output) { - this.output = output; - - alphabet = ((flags & URL_SAFE) == 0) ? DECODE : DECODE_WEBSAFE; - state = 0; - value = 0; - } - - /** - * @return an overestimate for the number of bytes {@code - * len} bytes could decode to. - */ - public int maxOutputSize(int len) { - return len * 3/4 + 10; - } - - /** - * Decode another block of input data. - * - * @return true if the state machine is still healthy. false if - * bad base-64 data has been detected in the input stream. - */ - public boolean process(byte[] input, int offset, int len, boolean finish) { - if (this.state == 6) return false; - - int p = offset; - len += offset; - - // Using local variables makes the decoder about 12% - // faster than if we manipulate the member variables in - // the loop. (Even alphabet makes a measurable - // difference, which is somewhat surprising to me since - // the member variable is final.) - int state = this.state; - int value = this.value; - int op = 0; - final byte[] output = this.output; - final int[] alphabet = this.alphabet; - - while (p < len) { - // Try the fast path: we're starting a new tuple and the - // next four bytes of the input stream are all data - // bytes. This corresponds to going through states - // 0-1-2-3-0. We expect to use this method for most of - // the data. - // - // If any of the next four bytes of input are non-data - // (whitespace, etc.), value will end up negative. (All - // the non-data values in decode are small negative - // numbers, so shifting any of them up and or'ing them - // together will result in a value with its top bit set.) - // - // You can remove this whole block and the output should - // be the same, just slower. - if (state == 0) { - while (p+4 <= len && - (value = ((alphabet[input[p] & 0xff] << 18) | - (alphabet[input[p+1] & 0xff] << 12) | - (alphabet[input[p+2] & 0xff] << 6) | - (alphabet[input[p+3] & 0xff]))) >= 0) { - output[op+2] = (byte) value; - output[op+1] = (byte) (value >> 8); - output[op] = (byte) (value >> 16); - op += 3; - p += 4; - } - if (p >= len) break; - } - - // The fast path isn't available -- either we've read a - // partial tuple, or the next four input bytes aren't all - // data, or whatever. Fall back to the slower state - // machine implementation. - - int d = alphabet[input[p++] & 0xff]; - - switch (state) { - case 0: - if (d >= 0) { - value = d; - ++state; - } else if (d != SKIP) { - this.state = 6; - return false; - } - break; - - case 1: - if (d >= 0) { - value = (value << 6) | d; - ++state; - } else if (d != SKIP) { - this.state = 6; - return false; - } - break; - - case 2: - if (d >= 0) { - value = (value << 6) | d; - ++state; - } else if (d == EQUALS) { - // Emit the last (partial) output tuple; - // expect exactly one more padding character. - output[op++] = (byte) (value >> 4); - state = 4; - } else if (d != SKIP) { - this.state = 6; - return false; - } - break; - - case 3: - if (d >= 0) { - // Emit the output triple and return to state 0. - value = (value << 6) | d; - output[op+2] = (byte) value; - output[op+1] = (byte) (value >> 8); - output[op] = (byte) (value >> 16); - op += 3; - state = 0; - } else if (d == EQUALS) { - // Emit the last (partial) output tuple; - // expect no further data or padding characters. - output[op+1] = (byte) (value >> 2); - output[op] = (byte) (value >> 10); - op += 2; - state = 5; - } else if (d != SKIP) { - this.state = 6; - return false; - } - break; - - case 4: - if (d == EQUALS) { - ++state; - } else if (d != SKIP) { - this.state = 6; - return false; - } - break; - - case 5: - if (d != SKIP) { - this.state = 6; - return false; - } - break; - } - } - - if (!finish) { - // We're out of input, but a future call could provide - // more. - this.state = state; - this.value = value; - this.op = op; - return true; - } - - // Done reading input. Now figure out where we are left in - // the state machine and finish up. - - switch (state) { - case 0: - // Output length is a multiple of three. Fine. - break; - case 1: - // Read one extra input byte, which isn't enough to - // make another output byte. Illegal. - this.state = 6; - return false; - case 2: - // Read two extra input bytes, enough to emit 1 more - // output byte. Fine. - output[op++] = (byte) (value >> 4); - break; - case 3: - // Read three extra input bytes, enough to emit 2 more - // output bytes. Fine. - output[op++] = (byte) (value >> 10); - output[op++] = (byte) (value >> 2); - break; - case 4: - // Read one padding '=' when we expected 2. Illegal. - this.state = 6; - return false; - case 5: - // Read all the padding '='s we expected and no more. - // Fine. - break; - } - - this.state = state; - this.op = op; - return true; - } - } - - // -------------------------------------------------------- - // encoding - // -------------------------------------------------------- - - /** - * Base64-encode the given data and return a newly allocated - * String with the result. - * - * @param input the data to encode - * @param flags controls certain features of the encoded output. - * Passing {@code DEFAULT} results in output that - * adheres to RFC 2045. - */ - public static String encodeToString(byte[] input, int flags) { - try { - return new String(encode(input, flags), "US-ASCII"); - } catch (UnsupportedEncodingException e) { - // US-ASCII is guaranteed to be available. - throw new AssertionError(e); - } - } - - /** - * Base64-encode the given data and return a newly allocated - * String with the result. - * - * @param input the data to encode - * @param offset the position within the input array at which to - * start - * @param len the number of bytes of input to encode - * @param flags controls certain features of the encoded output. - * Passing {@code DEFAULT} results in output that - * adheres to RFC 2045. - */ - public static String encodeToString(byte[] input, int offset, int len, int flags) { - try { - return new String(encode(input, offset, len, flags), "US-ASCII"); - } catch (UnsupportedEncodingException e) { - // US-ASCII is guaranteed to be available. - throw new AssertionError(e); - } - } - - /** - * Base64-encode the given data and return a newly allocated - * byte[] with the result. - * - * @param input the data to encode - * @param flags controls certain features of the encoded output. - * Passing {@code DEFAULT} results in output that - * adheres to RFC 2045. - */ - public static byte[] encode(byte[] input, int flags) { - return encode(input, 0, input.length, flags); - } - - /** - * Base64-encode the given data and return a newly allocated - * byte[] with the result. - * - * @param input the data to encode - * @param offset the position within the input array at which to - * start - * @param len the number of bytes of input to encode - * @param flags controls certain features of the encoded output. - * Passing {@code DEFAULT} results in output that - * adheres to RFC 2045. - */ - public static byte[] encode(byte[] input, int offset, int len, int flags) { - Encoder encoder = new Encoder(flags, null); - - // Compute the exact length of the array we will produce. - int output_len = len / 3 * 4; - - // Account for the tail of the data and the padding bytes, if any. - if (encoder.do_padding) { - if (len % 3 > 0) { - output_len += 4; - } - } else { - switch (len % 3) { - case 0: break; - case 1: output_len += 2; break; - case 2: output_len += 3; break; - } - } - - // Account for the newlines, if any. - if (encoder.do_newline && len > 0) { - output_len += (((len-1) / (3 * Encoder.LINE_GROUPS)) + 1) * - (encoder.do_cr ? 2 : 1); - } - - encoder.output = new byte[output_len]; - encoder.process(input, offset, len, true); - - assert encoder.op == output_len; - - return encoder.output; - } - - /* package */ static class Encoder extends Coder { - /** - * Emit a new line every this many output tuples. Corresponds to - * a 76-character line length (the maximum allowable according to - * RFC 2045). - */ - public static final int LINE_GROUPS = 19; - - /** - * Lookup table for turning Base64 alphabet positions (6 bits) - * into output bytes. - */ - private static final byte ENCODE[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', - }; - - /** - * Lookup table for turning Base64 alphabet positions (6 bits) - * into output bytes. - */ - private static final byte ENCODE_WEBSAFE[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_', - }; - - final private byte[] tail; - /* package */ int tailLen; - private int count; - - final public boolean do_padding; - final public boolean do_newline; - final public boolean do_cr; - final private byte[] alphabet; - - public Encoder(int flags, byte[] output) { - this.output = output; - - do_padding = (flags & NO_PADDING) == 0; - do_newline = (flags & NO_WRAP) == 0; - do_cr = (flags & CRLF) != 0; - alphabet = ((flags & URL_SAFE) == 0) ? ENCODE : ENCODE_WEBSAFE; - - tail = new byte[2]; - tailLen = 0; - - count = do_newline ? LINE_GROUPS : -1; - } - - /** - * @return an overestimate for the number of bytes {@code - * len} bytes could encode to. - */ - public int maxOutputSize(int len) { - return len * 8/5 + 10; - } - - public boolean process(byte[] input, int offset, int len, boolean finish) { - // Using local variables makes the encoder about 9% faster. - final byte[] alphabet = this.alphabet; - final byte[] output = this.output; - int op = 0; - int count = this.count; - - int p = offset; - len += offset; - int v = -1; - - // First we need to concatenate the tail of the previous call - // with any input bytes available now and see if we can empty - // the tail. - - switch (tailLen) { - case 0: - // There was no tail. - break; - - case 1: - if (p+2 <= len) { - // A 1-byte tail with at least 2 bytes of - // input available now. - v = ((tail[0] & 0xff) << 16) | - ((input[p++] & 0xff) << 8) | - (input[p++] & 0xff); - tailLen = 0; - }; - break; - - case 2: - if (p+1 <= len) { - // A 2-byte tail with at least 1 byte of input. - v = ((tail[0] & 0xff) << 16) | - ((tail[1] & 0xff) << 8) | - (input[p++] & 0xff); - tailLen = 0; - } - break; - } - - if (v != -1) { - output[op++] = alphabet[(v >> 18) & 0x3f]; - output[op++] = alphabet[(v >> 12) & 0x3f]; - output[op++] = alphabet[(v >> 6) & 0x3f]; - output[op++] = alphabet[v & 0x3f]; - if (--count == 0) { - if (do_cr) output[op++] = '\r'; - output[op++] = '\n'; - count = LINE_GROUPS; - } - } - - // At this point either there is no tail, or there are fewer - // than 3 bytes of input available. - - // The main loop, turning 3 input bytes into 4 output bytes on - // each iteration. - while (p+3 <= len) { - v = ((input[p] & 0xff) << 16) | - ((input[p+1] & 0xff) << 8) | - (input[p+2] & 0xff); - output[op] = alphabet[(v >> 18) & 0x3f]; - output[op+1] = alphabet[(v >> 12) & 0x3f]; - output[op+2] = alphabet[(v >> 6) & 0x3f]; - output[op+3] = alphabet[v & 0x3f]; - p += 3; - op += 4; - if (--count == 0) { - if (do_cr) output[op++] = '\r'; - output[op++] = '\n'; - count = LINE_GROUPS; - } - } - - if (finish) { - // Finish up the tail of the input. Note that we need to - // consume any bytes in tail before any bytes - // remaining in input; there should be at most two bytes - // total. - - if (p-tailLen == len-1) { - int t = 0; - v = ((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 4; - tailLen -= t; - output[op++] = alphabet[(v >> 6) & 0x3f]; - output[op++] = alphabet[v & 0x3f]; - if (do_padding) { - output[op++] = '='; - output[op++] = '='; - } - if (do_newline) { - if (do_cr) output[op++] = '\r'; - output[op++] = '\n'; - } - } else if (p-tailLen == len-2) { - int t = 0; - v = (((tailLen > 1 ? tail[t++] : input[p++]) & 0xff) << 10) | - (((tailLen > 0 ? tail[t++] : input[p++]) & 0xff) << 2); - tailLen -= t; - output[op++] = alphabet[(v >> 12) & 0x3f]; - output[op++] = alphabet[(v >> 6) & 0x3f]; - output[op++] = alphabet[v & 0x3f]; - if (do_padding) { - output[op++] = '='; - } - if (do_newline) { - if (do_cr) output[op++] = '\r'; - output[op++] = '\n'; - } - } else if (do_newline && op > 0 && count != LINE_GROUPS) { - if (do_cr) output[op++] = '\r'; - output[op++] = '\n'; - } - - assert tailLen == 0; - assert p == len; - } else { - // Save the leftovers in tail to be consumed on the next - // call to encodeInternal. - - if (p == len-1) { - tail[tailLen++] = input[p]; - } else if (p == len-2) { - tail[tailLen++] = input[p]; - tail[tailLen++] = input[p+1]; - } - } - - this.op = op; - this.count = count; - - return true; - } - } -} diff --git a/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 b/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 deleted file mode 100644 index c8a8c53b2c..0000000000 --- a/third-party-licenses/AOSP-APACHE-LICENSE-V2.0 +++ /dev/null @@ -1,228 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - - - -UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE - -Unicode Data Files include all data files under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, -and http://www.unicode.org/cldr/data/ . Unicode Software includes any -source code published in the Unicode Standard or under the directories -http://www.unicode.org/Public/, http://www.unicode.org/reports/, and -http://www.unicode.org/cldr/data/. - -NOTICE TO USER: Carefully read the following legal agreement. BY -DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA -FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY -ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF -THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, -DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. - -COPYRIGHT AND PERMISSION NOTICE - -Copyright © 1991-2008 Unicode, Inc. All rights reserved. Distributed -under the Terms of Use in http://www.unicode.org/copyright.html. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Unicode data files and any associated documentation (the -"Data Files") or Unicode software and any associated documentation (the -"Software") to deal in the Data Files or Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, and/or sell copies of the Data Files or Software, -and to permit persons to whom the Data Files or Software are furnished to -do so, provided that (a) the above copyright notice(s) and this permission -notice appear with all copies of the Data Files or Software, (b) both the -above copyright notice(s) and this permission notice appear in associated -documentation, and (c) there is clear notice in each modified Data File -or in the Software as well as in the documentation associated with the -Data File(s) or Software that the data or software has been modified. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS -INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT -OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE -OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE -OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder -shall not be used in advertising or otherwise to promote the sale, use -or other dealings in these Data Files or Software without prior written -authorization of the copyright holder. From 7c8035483ded91fe4da68a385ee9763bc8226ceb Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sat, 23 Mar 2019 23:40:39 +0530 Subject: [PATCH 11/15] Create Base64Utils to work around alleged legacy apache commons codec usage in Android --- .../linkHandler/YoutubeSearchQueryHandlerFactory.java | 3 ++- .../schabi/newpipe/extractor/utils/Base64Utils.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 670666533f..b534674a9a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -3,6 +3,7 @@ import org.apache.commons.codec.binary.Base64; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Base64Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -168,7 +169,7 @@ private String getFilterQueryParams(List contentFilters, String sortFilt if (returnList.isEmpty()) { return null; } - return URLEncoder.encode(Base64.encodeBase64String(convert(returnList)), "UTF-8"); + return URLEncoder.encode(Base64Utils.encodeBase64String(convert(returnList)), "UTF-8"); } private List getContentFiltersQueryParams(List contentFilter) throws IllegalArgumentException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java new file mode 100644 index 0000000000..5f646e8d5f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java @@ -0,0 +1,10 @@ +package org.schabi.newpipe.extractor.utils; + +import org.apache.commons.codec.Charsets; +import org.apache.commons.codec.binary.Base64; + +public class Base64Utils { + public static String encodeBase64String(byte[] bytes) { + return new String(Base64.encodeBase64(bytes), Charsets.UTF_8); + } +} From 8152ccf4fa15816e5995b88451eec9b5e3d47482 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Sun, 24 Mar 2019 00:10:20 +0530 Subject: [PATCH 12/15] Add comment explaining purpose of Base64Utils and make bytes array argument non null --- .../newpipe/extractor/utils/Base64Utils.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java index 5f646e8d5f..ea393fe812 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Base64Utils.java @@ -3,8 +3,22 @@ import org.apache.commons.codec.Charsets; import org.apache.commons.codec.binary.Base64; +import javax.annotation.Nonnull; + public class Base64Utils { - public static String encodeBase64String(byte[] bytes) { + /* + Using Base64.encodeBase64String() throws a NoSuchMethodError when + using it on Android. The reason behind this seems to be that + Android already includes an older version of Apache Commons Codec + which does not have this method, as per below SO thread: + + https://stackoverflow.com/questions/2047706/apache-commons-codec-with-android-could-not-find-method + + Hence, encodeBase64 method has been used which is an older method. + Creating a String out of it is exactly what was being done in the + library too, so that has been pulled in here. + */ + public static String encodeBase64String(@Nonnull byte[] bytes) { return new String(Base64.encodeBase64(bytes), Charsets.UTF_8); } } From 55b126ae0e0404f25577e6eb35e61ab51f35ed82 Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Tue, 9 Apr 2019 12:11:34 +0530 Subject: [PATCH 13/15] Altered test cases to test all filters and sorters individually and use correct titles instead of name when matching elements. --- .../YoutubeSearchQueryHandlerFactory.java | 20 +++-- .../youtube/search/YoutubeSearchQHTest.java | 83 ++++++++----------- 2 files changed, 48 insertions(+), 55 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index b534674a9a..48b848dd61 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -40,22 +40,22 @@ public enum Filter { Movie("Movie", FilterType.Content, new byte[]{0x04}), Show("Show", FilterType.Content, new byte[]{0x05}), - Hour("Hour", FilterType.Time, new byte[]{0x01}), + Hour("Last hour", FilterType.Time, new byte[]{0x01}), Today("Today", FilterType.Time, new byte[]{0x02}), - Week("Week", FilterType.Time, new byte[]{0x03}), - Month("Month", FilterType.Time, new byte[]{0x04}), - Year("Year", FilterType.Time, new byte[]{0x05}), + Week("This week", FilterType.Time, new byte[]{0x03}), + Month("This month", FilterType.Time, new byte[]{0x04}), + Year("This year", FilterType.Time, new byte[]{0x05}), Short("Short", FilterType.Duration, new byte[]{0x01}), Long("Long", FilterType.Duration, new byte[]{0x02}), HD("HD", FilterType.Feature, new byte[]{0x20, 0x01}), - Subtitles("Subtitles", FilterType.Feature, new byte[]{0x28, 0x01}), + Subtitles("Subtitles/CC", FilterType.Feature, new byte[]{0x28, 0x01}), CreativeCommons("Creative Commons", FilterType.Feature, new byte[]{0x30, 0x01}), ThreeDimensional("3D", FilterType.Feature, new byte[]{0x38, 0x01}), Live("Live", FilterType.Feature, new byte[]{0x40, 0x01}), Purchased("Purchased", FilterType.Feature, new byte[]{0x48, 0x01}), - FourK("4k", FilterType.Feature, new byte[]{0x70, 0x01}), + FourK("4K", FilterType.Feature, new byte[]{0x70, 0x01}), ThreeSixty("360", FilterType.Feature, new byte[]{0x78, 0x01}), Location("Location", FilterType.Feature, new byte[]{(byte) 0xb8, 0x01, 0x01}), HDR("HDR", FilterType.Feature, new byte[]{(byte) 0xc8, 0x01, 0x01}); @@ -88,8 +88,8 @@ public enum SorterType { public enum Sorter { Relevance("Relevance", SorterType.Default, (byte) 0x00), Rating("Rating", SorterType.Default, (byte) 0x01), - Upload_Date("Upload_Date", SorterType.Default, (byte) 0x02), - View_Count("View_Count", SorterType.Default, (byte) 0x03); + Upload_Date("Upload date", SorterType.Default, (byte) 0x02), + View_Count("View count", SorterType.Default, (byte) 0x03); private final String title; private final SorterType type; @@ -100,6 +100,10 @@ public enum Sorter { this.type = type; this.value = value; } + + public String getTitle() { + return title; + } } public static YoutubeSearchQueryHandlerFactory getInstance() { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 6c80d3e2db..ee8ea07e32 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -50,38 +50,45 @@ public void testGetContentFilter() throws Exception { } @Test - public void testWithChannelContentFilter() throws Exception { - String url = getYouTubeDefaultSearchQueryHandler( - Collections.singletonList(Filter.Channel.name()), - "" - ).getUrl(); - String html = getHtml(url); - Document document = Jsoup.parse(html); - Elements filterList = getFilterList(document); - Element matchingElement = getMatchingElement( - Filter.Channel.name(), - filterList - ); - if (matchingElement == null) { - fail("Channel filter has not been selected"); + public void testWithAllContentFiltersIndividually() throws Exception { + for(Filter filter : Filter.values()) { + if(filter == Filter.All) { + continue; + } + String url = getYouTubeDefaultSearchQueryHandler( + Collections.singletonList(filter.name()), + "" + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element matchingElement = getMatchingElement( + filter.getTitle(), + filterList + ); + if (matchingElement == null) { + fail(filter.name() + " filter has not been selected"); + } } } @Test - public void testWith360ContentFilterAndNoSortFilter() throws Exception { - String url = getYouTubeDefaultSearchQueryHandler( - Collections.singletonList(Filter.ThreeSixty.name()), - "" - ).getUrl(); - String html = getHtml(url); - Document document = Jsoup.parse(html); - Elements filterList = getFilterList(document); - Element matchingElement = getMatchingElement( - Filter.ThreeSixty.getTitle(), - filterList - ); - if (matchingElement == null) { - fail("360 filter has not been selected"); + public void testWithAllSortFiltersIndividually() throws Exception { + for(Sorter sorter : Sorter.values()) { + String url = getYouTubeDefaultSearchQueryHandler( + Collections.emptyList(), + sorter.name() + ).getUrl(); + String html = getHtml(url); + Document document = Jsoup.parse(html); + Elements filterList = getFilterList(document); + Element matchingElement = getMatchingElement( + sorter.getTitle(), + filterList + ); + if (matchingElement == null) { + fail("Rating sorter has not been selected"); + } } } @@ -99,7 +106,7 @@ public void testWith360ContentFilterAndRatingSortFilter() throws Exception { filterList ); Element ratingMatchingElement = getMatchingElement( - Sorter.Rating.name(), + Sorter.Rating.getTitle(), filterList ); if (threeSixtyMatchingElement == null) { @@ -110,24 +117,6 @@ public void testWith360ContentFilterAndRatingSortFilter() throws Exception { } } - @Test - public void testWithRatingSortFilter() throws Exception { - String url = getYouTubeDefaultSearchQueryHandler( - Collections.emptyList(), - Sorter.Rating.name() - ).getUrl(); - String html = getHtml(url); - Document document = Jsoup.parse(html); - Elements filterList = getFilterList(document); - Element matchingElement = getMatchingElement( - Sorter.Rating.name(), - filterList - ); - if (matchingElement == null) { - fail("Rating sorter has not been selected"); - } - } - @Test public void testWithVideoAndShortContentFilter() throws Exception { String url = getYouTubeDefaultSearchQueryHandler( From 4cfd07bba07ab87420e950141b3708a04146710e Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Mon, 30 Dec 2019 23:12:43 +0530 Subject: [PATCH 14/15] Undo unnecessary change made to gitignore file --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index cac877d61c..bd009fd87d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,3 @@ gradle-app.setting # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties - -.java-version \ No newline at end of file From 68f95471fdc8dc1989676f1e6c05f8c6e7d74b6e Mon Sep 17 00:00:00 2001 From: Vishan Seru Date: Mon, 30 Dec 2019 23:13:15 +0530 Subject: [PATCH 15/15] Change Movie filter name to Film and Show filter name to Programme --- .../youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java index 48b848dd61..135daa301a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java @@ -37,8 +37,8 @@ public enum Filter { Video("Video", FilterType.Content, new byte[]{0x01}), Channel("Channel", FilterType.Content, new byte[]{0x02}), Playlist("Playlist", FilterType.Content, new byte[]{0x03}), - Movie("Movie", FilterType.Content, new byte[]{0x04}), - Show("Show", FilterType.Content, new byte[]{0x05}), + Film("Film", FilterType.Content, new byte[]{0x04}), + Programme("Programme", FilterType.Content, new byte[]{0x05}), Hour("Last hour", FilterType.Time, new byte[]{0x01}), Today("Today", FilterType.Time, new byte[]{0x02}),