Experimenting multiprocessing with multi Threads requesting to the url in python.


Finally i got all the url html responce correctly it is working fine as expected.With multiprocessing and multi Threads.



main.py

def calc_square(number):
   time.sleep(3)
   print('Square:' , number * number)
   result = number * number
   print(os.getpid())

print(result)

def calc_quad(number):
   print(os.getpid())

print('Quad:' , number * number * number * number)


def test1():
    print(os.getpid())
    print('test1 :' ,)

def test2():
    print(os.getpid())
    print('test2 :' ,)

def test3():
    print(os.getpid())
    print('test3 :' ,)

if __name__ == '__main__':


   number = 7
   start = time.time()
   result = None

   # p1 = multiprocessing.Process(target=calc_square, args=(number,))
   # p2 = multiprocessing.Process(target=calc_quad, args=(number,))
     p3 = multiprocessing.Process(target=run_test1)
     p4 = multiprocessing.Process(target=run_test)
     p5 = multiprocessing.Process(target=test3)


    # p1.start()
    # p2.start()
    p3.start()
    p4.start()
     p5.start()

    # p1.join()
   # p2.join()
    p3.join()
    p4.join()
    p5.join()

   end = time.time()
   print(f'Time taken {end - start}')
test.py
import os
import time

from Utility.NetworkRequest import getHtml_Str_From_Url
from Utility.Utility import create_thread

link_list = ['https://tutorialedge.net/python/concurrency/python-threadpoolexecutor- 
  tutorial/'
       , 'https://www.machinelearningplus.com/python/parallel-processing-python/']

def run_test():

  # count = 0

  thread = create_thread(run_other)
  thread.start()
  print(hash(thread))


def run_other():
   count = 0

  while len(link_list) > count:
      print("from test")
      print("process id:- " + str(os.getpid()))
      print("links: - " + str(link_list[count]))
      getHtml = getHtml_Str_From_Url(link_list[count])
      print(getHtml)
      count += 1
save.py
import os
import time

from Utility.NetworkRequest import getHtml_Str_From_Url
from Utility.Utility import create_thread

 link_list = ['https://realpython.com/intro-to-python- 
   threading/','https://pymotw.com/3/concurrent.futures/']

def run_test1():

   thread = create_thread(run_other)
   thread.start()
   print(hash(thread))

def run_other():
    count = 0

    while len(link_list) > count:
        print("from save")
        print("process id:- " + str(os.getpid()))
        print("links: - " + str(link_list[count]))
        getHtml = getHtml_Str_From_Url(link_list[count])
        print(getHtml)
        count += 1

Comments

Popular posts from this blog

Android Studio 3.5 or above usb device not detected in Android Studio or Unable to detect adb version, exit value: 0xc0000135. ADB depends on the Windows Universal C Runtime, which is usually installed by default via Windows Update. You may need to manually fetch and install the runtime package here: https://support.microsoft.com/en-ca/help/2999226/update-for-universal-c-runtime-in-windows

Lint found fatal errors while assembling a release target.

ViewModelProviders.of(this) is deprecated what to use instead ViewModelProviders.of(this) OR if you don't want to use ViewmodelFactory OR ViewModelProviders error.